brute 5.0.4 → 5.1.0

data/lib/brute/eval.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "brute"
+
+require_relative "eval/case"
+require_relative "eval/suite"
+require_relative "eval/transcript"
+require_relative "eval/world"
+
+module Brute
+  # Evaluating an agent, rather than testing a middleware.
+  #
+  # A spec asks whether one layer does what it says. An eval asks whether the
+  # whole assembled agent -- its system prompt, its tools, the model behind
+  # it -- behaves. That answer is not a boolean about code: it is a real turn,
+  # against a real model, graded on what the turn DID.
+  #
+  #   CASES = [
+  #     Brute::Eval::Case.new(
+  #       "searches for what it cannot know",
+  #       said:     "what did the Bank of England do yesterday?",
+  #       stubs:    { "search" => RATE_DECISION },
+  #       calls:    { "search" => { "query" => /bank|rate/i } },
+  #       mentions: %w[4.25],
+  #     ),
+  #     Brute::Eval::Case.new(
+  #       "does not search for what it already knows",
+  #       said:   "how many minutes are there in an hour?",
+  #       never:  %w[search],
+  #       budget: Brute::Eval::Budget.new(iterations: 2, tool_calls: 0),
+  #     ),
+  #   ]
+  #
+  #   exit(Brute::Eval::Suite.new(agent: "agent.ru", cases: CASES).run)
+  #
+  # Everything the harness sees comes off the agent's own hooks -- the same
+  # registry any other subscriber uses -- so the agent under evaluation is the
+  # agent that ships: no eval-only middleware, no branch in agent.ru. The
+  # model and the tool schemas are real; the tools themselves answer from the
+  # case's stubs, installed on :before_tool, which answers a call without
+  # executing it.
+  #
+  # Where a case wakes up is the World's business, and a deployment that
+  # delivers what was said through an inbox, a queue or a room subclasses it.
+  module Eval
+  end
+end

data/lib/brute/hooks.rb

@@ -42,6 +42,15 @@ module Brute
   #                             completion middleware then emits one of
   #                             :faraday_error, :open_router_server_error or
   #                             :standard_error with the exception as an extra
+  #   :compact_duration       → env, started, finished, the compactor: one
+  #                             strategy's attempt is this event's block, so
+  #                             the strategy that spends a model call is
+  #                             visible, and one that declined is still timed
+  #   :compacted              → env, {strategy:, before:, after:} — a
+  #                             compaction strategy rewrote env[:messages];
+  #                             what it replaced is already gone, so an
+  #                             application that keeps a transcript preserves
+  #                             it here
   #   :before_tool            → env, call env {name:, arguments:, result:,
   #                             denied:, events:, metadata:, turn_env:} —
   #                             mutate :arguments to rewrite the call, or set
@@ -75,6 +84,8 @@ module Brute
     FARADAY_ERROR_EVENT = :faraday_error
     OPEN_ROUTER_SERVER_ERROR_EVENT = :open_router_server_error
     STANDARD_ERROR_EVENT = :standard_error
+    COMPACT_DURATION_EVENT = :compact_duration
+    COMPACTED_EVENT = :compacted
     BEFORE_TOOL_EVENT = :before_tool
     APPROVE_TOOL_EVENT = :approve_tool
     TOOL_DURATION_EVENT = :tool_duration

data/lib/brute/middleware/000_base.rb

@@ -55,7 +55,7 @@ describe "brute/middleware/000_base" do
 
     # Every middleware in the chain descends from it.
     Brute::Middleware::SystemPrompt.ancestors.should.include Brute::Middleware::Base
-    Brute::Middleware::ToolPipeline.ancestors.should.include Brute::Middleware::Base
+    Brute::Middleware::DefaultToolPipeline.ancestors.should.include Brute::Middleware::Base
     Brute::Middleware::Loop::ToolResult.ancestors.should.include Brute::Middleware::Base
   end
 end

data/lib/brute/middleware/001_slash_commands.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "brute"
+
+module Brute
+  module Middleware
+    # The head of every agent chain, put there by the builder itself rather
+    # than by a `use` anyone writes. It switches on what was just said: the
+    # newest message, and only when it is a user message, is offered to each
+    # of env[:commands]'s checks in turn -- the commands registered with
+    # `AgentPipeline#map`, which `start` puts there. The first check that
+    # passes has its block run here, before the rest of the stack.
+    #
+    #   Brute.agent
+    #     .map("/compact") { |env| ... }
+    #     .run(->(env) { ... })
+    #
+    # A command's block is a middleware, so what it leaves in env is what
+    # the rest of the chain works on.
+    class SlashCommands < Brute::Middleware::Base
+      def call(env)
+        matched(env)&.call(env)
+        @app.call(env)
+      end
+
+      private
+
+        # The block of the first command whose check passes on what was just
+        # said -- and nothing at all unless the room said it, so not the
+        # assistant and not an empty log.
+        def matched(env)
+          message = Array(env[:messages]).last
+
+          if message.respond_to?(:role) && message.role == :user
+            _check, block = Array(env[:commands]).find { |check, _block| check.call(message.content.to_s) }
+            block
+          end
+        end
+    end
+  end
+end
+
+__END__
+
+describe "brute/middleware/001_slash_commands" do
+  require "brute/messages"
+
+  it "runs the first of env[:commands] whose check matches what was just said" do
+    ran = []
+    commands = [
+      [->(said) { said.start_with?("/compact") }, ->(env) { ran << [:compact, env[:messages].last.content] }],
+      [->(said) { said.length > 20 }, ->(_env) { ran << :long }],
+    ]
+
+    reached = []
+    inner = ->(env) { reached << env[:messages].last&.content; env }
+    middleware = Brute::Middleware::SlashCommands.new(inner)
+
+    def turn(commands, said, role = :user)
+      { commands: commands, messages: Brute.log.tap { |log| log << Brute::Message.new(role: role, content: said) } }
+    end
+
+    # The matching command runs before the stack below it.
+    middleware.call(turn(commands, "/compact keep the notes"))
+    ran.should == [[:compact, "/compact keep the notes"]]
+    reached.should == ["/compact keep the notes"]
+
+    # Checks are tried in order, and only the first to pass runs.
+    ran.clear
+    middleware.call(turn(commands, "something else, at some length"))
+    ran.should == [:long]
+
+    # Nothing matches, nothing runs -- and the turn carries on regardless.
+    ran.clear
+    reached.clear
+    middleware.call(turn(commands, "short"))
+    middleware.call(turn(commands, "/compact", :assistant))
+    middleware.call({ commands: commands, messages: Brute.log })
+    ran.should.be.empty
+    reached.should == ["short", "/compact", nil]
+
+    # A turn carrying no commands at all still runs, and does nothing.
+    ran.clear
+    middleware.call({ messages: Brute.log.tap { |log| log.user("/compact") } })
+    ran.should.be.empty
+  end
+end

data/lib/brute/middleware/008_checkpoint.rb

@@ -16,7 +16,7 @@ module Brute
     #   use Brute::Middleware::Loop::ToolResult
     #   use Brute::Middleware::Checkpoint, path: "tmp/checkpoints.jsonl"
     #   use Brute::Middleware::MaxIterations
-    #   use Brute::Middleware::ToolPipeline, tools: Brute::Tools::ALL
+    #   use Brute::Middleware::DefaultToolPipeline, tools: Brute::Tools::ALL
     #
     # The store is just a JSONL log of snapshots — one line per checkpoint,
     # each carrying the full message log plus its own id and the id of the

data/lib/brute/middleware/040_compaction_check.rb

@@ -2,150 +2,35 @@
 
 require "bundler/setup"
 require "brute"
+require "gem_kit"
 
 module Brute
   module Middleware
-    # Checks context size after each LLM call and triggers compaction
-    # when thresholds are exceeded.
+    # The old compaction trigger, kept working while it is deprecated.
+    # DefaultCompactionPipeline is the layer that does the job.
     #
-    # It should add a compaction event to the logs with the context token
-    # total listed... this way a model that supports extra context can
-    # include the compaction as well as the previous messages...
+    # It passes the turn through and compacts nothing, which is what it always
+    # did -- and it is deliberately not a subclass of its replacement, because
+    # inheriting would make a layer that did nothing suddenly start giving up
+    # context. A deprecation warns about a name; it does not change what the
+    # code under that name does.
     #
-    # Or an LLM that doesn't support it can just use the messages
-    # that come after the compaction
+    # Whatever it was configured with is accepted and ignored, so existing
+    # `use` lines keep parsing until they are moved over:
+    #
+    #   use Brute::Middleware::DefaultCompactionPipeline,
+    #     window:     200_000,
+    #     summariser: Brute::Completion::OpenRouter.new(config: { access_token: key })
     #
     class CompactionCheck < Brute::Middleware::Base
-      def initialize(app, compactor: nil, system_prompt:, **compactor_opts)
-        @app = app
-        @compactor = compactor
-        @compactor_opts = compactor_opts
-        @system_prompt = system_prompt
-      end
-
-      def call(env)
-        #@compactor ||= Compactor.new(env[:provider], **@compactor_opts)
+      extend GemKit::Deprecate
+      superseded_by "Brute::Middleware::DefaultCompactionPipeline", "6.0"
 
-        #messages = env[:messages]
-        #usage = env[:metadata].dig(:tokens, :last_call)
-
-        #if @compactor.should_compact?(messages, usage: usage)
-        #  result = @compactor.compact(messages)
-        #  if result
-        #    summary_text, _recent = result
-        #    env[:metadata][:compaction] = {
-        #      messages_before: messages.size,
-        #      timestamp: Time.now.iso8601,
-        #    }
-        #    # Replace the message history with the summary
-        #    env[:messages] = [
-        #      Brute::Message.new(role: :system, content: @system_prompt),
-        #      Brute::Message.new(role: :user, content: "[Previous conversation summary]\n\n#{summary_text}"),
-        #    ]
-        #  end
-        #end
-
-        @app.call(env)
+      def initialize(app, **_options)
+        @app = app
       end
 
-      # Context compaction service. When the conversation grows past configurable
-      # thresholds, older messages are summarized into a condensed form and the
-      # original messages are dropped, keeping the context window manageable.
-      class Compactor
-        DEFAULTS = {
-          token_threshold: 100_000,   # Compact when estimated tokens exceed this
-          message_threshold: 200,     # Compact when message count exceeds this
-          retention_window: 6,        # Minimum recent messages to always keep
-          summary_model: nil,         # Model for summarization (uses agent's model if nil)
-        }.freeze
-
-        attr_reader :config
-
-        def initialize(provider, **opts)
-          @provider = provider
-          @config = DEFAULTS.merge(opts)
-        end
-
-        # Check whether compaction should run based on current context state.
-        def should_compact?(messages, usage: nil)
-          return true if messages.size > @config[:message_threshold]
-          return true if usage && (usage[:total] || 0) > @config[:token_threshold]
-          false
-        end
-
-        # Compact the message history by summarizing older messages.
-        #
-        # Returns [summary_message, kept_messages] — the caller rebuilds
-        # the context from these.
-        def compact(messages)
-          total = messages.size
-          keep_count = [@config[:retention_window], total].min
-          return nil if total <= keep_count
-
-          old_messages = messages[0...(total - keep_count)]
-          recent_messages = messages[(total - keep_count)..]
-
-          summary_text = summarize(old_messages)
-
-          [summary_text, recent_messages]
-        end
-
-        private
-
-        def summarize(messages)
-          # Build a condensed representation of the conversation for the summarizer
-          conversation_text = messages.map { |m|
-            role = if m.respond_to?(:role)
-              m.role.to_s
-            else
-              "unknown"
-            end
-            content = if m.respond_to?(:content)
-              m.content.to_s[0..1000]
-            else
-              m.to_s[0..1000]
-            end
-
-            # Include tool call info for assistant messages
-            tool_info = ""
-            if m.respond_to?(:functions) && m.functions&.any?
-              calls = m.functions.map { |f| "#{f.name}(#{f.arguments.to_s[0..200]})" }
-              tool_info = " [tools: #{calls.join(", ")}]"
-            end
-
-            "#{role}:#{tool_info} #{content}"
-          }.join("\n---\n")
-
-          prompt = <<~PROMPT
-            Summarize this conversation history for context continuity. The summary will replace
-            these messages in the context window, so include everything the agent needs to continue
-            working effectively.
-
-            Structure your summary as:
-            ## Goal
-            What the user asked for.
-
-            ## Progress
-            - Files read, created, or modified (list paths)
-            - Commands executed and their outcomes
-            - Key decisions made
-
-            ## Current State
-            Where things stand right now — what's done and what remains.
-
-            ## Next Steps
-            What should happen next based on the conversation.
-
-            ---
-            CONVERSATION:
-            #{conversation_text}
-          PROMPT
-
-          model = @config[:summary_model] || "claude-sonnet-4-20250514"
-          res = @provider.complete(prompt, model: model)
-          res.content
-        end
-      end
+      def call(env) = @app.call(env)
     end
   end
 end
@@ -153,5 +38,36 @@ end
 __END__
 
 describe "brute/middleware/040_compaction_check" do
-  # not implemented
+  it "passes the turn through as it always did, and says what to use instead" do
+    warned = []
+    original = GemKit::Deprecate.method(:warn)
+    GemKit::Deprecate.define_singleton_method(:warn) { |message| warned << message }
+
+    begin
+      # Whatever it was configured with is taken and ignored, so an existing
+      # `use` line keeps parsing.
+      layer = Brute::Middleware::CompactionCheck.new(
+        ->(env) { env[:messages].assistant("answered") },
+        system_prompt: "you are helpful",
+        token_threshold: 100
+      )
+
+      env = { messages: Brute.log }
+      layer.call(env)
+      env[:messages].last.content.should == "answered"
+    ensure
+      GemKit::Deprecate.define_singleton_method(:warn, original)
+    end
+
+    # Not a subclass of its replacement: a layer that compacted nothing must
+    # not start compacting because its name was deprecated.
+    Brute::Middleware::CompactionCheck.ancestors
+      .should.not.include Brute::Middleware::DefaultCompactionPipeline
+
+    warned.first.should.include "Brute::Middleware::DefaultCompactionPipeline"
+
+    declared = GemKit::Deprecate.registry.find { |entry| entry.name == "Brute::Middleware::CompactionCheck" }
+    declared.replacement.should == "Brute::Middleware::DefaultCompactionPipeline"
+    declared.removed_in.should == Gem::Version.new("6.0")
+  end
 end