brute 5.0.5 → 5.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 146d42f66c911ce3ce45384eaeadd96bb877befe2740a417f405fb693c73acc2
-  data.tar.gz: 439445efa583943587f1d70d03c626a6d34e58d40509f03d3d3ce33c9edf03c9
+  metadata.gz: fd46c42475fdbdaef5ec4f22cd15ff43d6d1bcf2dc2c37c744fb4c6dbbdc2289
+  data.tar.gz: 2ef42a57205047a0a0f0783a736a333ef7ee0d1fcbe583c0dbf7b97c7e2ba514
 SHA512:
-  metadata.gz: 36ed02483e9838d2d71272bf471f5ffa025cf980b56b54c576873c011c85085c89320f513a00002acb814fc01c333f8882c796af2a900828b5532b26aea5c800
-  data.tar.gz: 925cc550884beadf57fbc631f7b796afc5484af8fea604e6b2cad19459159c2a664eff45a7429b0c385bc894320867ad0d12946ed26ccd42b7086b9c93b66926
+  metadata.gz: cf83163546bc17c6ed562581067db5e6341088d2746fac3476ac56fcfeb2ac5230d3a7317a02c94a42a9b3ba6c43203578ecc995517d561741d788946b168064
+  data.tar.gz: 9f42fb22bc76a611043dadbcd2acb6f6b4862efda77e31af58c862d0ad1cb99587b46169055cd8ccd74936eeef8d1af2b42faa14b14fa6b6548c2ccf8b0dbdf0

data/lib/brute/compaction/middleware/sliding_window.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "brute"
+
+module Brute
+  module Compaction
+    module Middleware
+      # When rewriting the tool output was not enough, whole stretches of the
+      # conversation go.
+      #
+      # The instructions and the message the current task hangs off are never
+      # given up. Past that, the oldest complete turns go first; only once every
+      # turn is gone does the current task start losing its own oldest steps, and
+      # never the newest.
+      #
+      # A note stands where the removed messages were, so the model can see that
+      # something was there rather than quietly reasoning from a gap.
+      class SlidingWindow < Strategy
+        NOTE = "%d earlier messages were dropped to free context. They cannot be recovered."
+
+        def initialize(app = nil, keep_steps: 1)
+          @app = app
+          @keep_steps = keep_steps
+        end
+
+        def strategy = "sliding_window"
+
+        def rewrite(messages, target:)
+          kept, note_index, dropped = split(messages, target)
+
+          # Swapping one note for another frees nothing. The base would refuse
+          # it anyway; declining here says why.
+          if dropped.any? && !dropped.all? { |index| Brute::Compaction::Transcript.marked?(messages[index], strategy) }
+            note = Brute::Message.new(
+              role: :user,
+              content: Brute::Compaction::Transcript.mark(strategy, format(NOTE, dropped.length)),
+            )
+            [*kept[...note_index], note, *kept[note_index..]]
+          end
+        end
+
+        private
+
+          def split(messages, target)
+            system_end = Brute::Compaction::Transcript.system_end(messages)
+            anchor     = Array(Brute::Compaction::Transcript.task_index(messages))
+            history    = Brute::Compaction::Transcript.turns(messages, from: system_end, to: anchor.first || system_end).map(&:to_a)
+            steps      = Brute::Compaction::Transcript.steps(messages, from: Brute::Compaction::Transcript.step_start(messages)).map(&:to_a)
+
+            budget = target - tokens(Brute::Compaction::Transcript.at(messages, [*0...system_end, *anchor]))
+            first_turn, first_step = frontier(
+              messages,
+              history,
+              steps,
+              budget,
+            )
+
+            turn_indices = history[first_turn..].flatten
+            step_indices = steps[first_step..].flatten
+            kept = [*0...system_end, *turn_indices, *anchor, *step_indices]
+
+            # The note goes where the messages it stands for used to sit: after
+            # the instructions when only history went, after the anchor when the
+            # task's own steps did.
+            note_index = system_end
+            if first_step.positive?
+              note_index = system_end + turn_indices.length + anchor.length
+            end
+
+            [Brute::Compaction::Transcript.at(messages, kept), note_index, (0...messages.length).to_a - kept]
+          end
+
+          # How far into the history, and into the task's own steps, keeping has
+          # to start for what remains to fit.
+          def frontier(messages, history, steps, budget)
+            spent = tokens(Brute::Compaction::Transcript.at(messages, steps.flatten))
+
+            if spent > budget
+              # The task alone overruns, so all history goes and its oldest steps
+              # follow -- down to the newest, which stay regardless.
+              floor = [steps.length - @keep_steps, 0].max
+              [history.length, [first_kept(messages, steps, budget), floor].min]
+            else
+              [first_kept(messages, history, budget - spent), 0]
+            end
+          end
+
+          # Newest group backwards, stopping at the first that does not fit.
+          def first_kept(messages, groups, budget)
+            position = groups.length
+            remaining = budget
+
+            while position.positive?
+              cost = tokens(Brute::Compaction::Transcript.at(messages, groups[position - 1]))
+
+              if cost > remaining
+                break
+              end
+
+              remaining -= cost
+              position -= 1
+            end
+
+            position
+          end
+      end
+    end
+  end
+end
+
+__END__
+
+describe "brute/compaction/middleware/sliding_window" do
+  def said(role, content) = Brute::Message.new(role: role, content: content)
+
+  def compacted(conversation, target, **options)
+    env = { conversation: conversation, target: target, applied: [], events: [] }
+    Brute::Compaction::Middleware::SlidingWindow.new(->(e) { e }, **options).call(env)
+    env
+  end
+
+  it "gives up the oldest turns first, then the task's oldest steps, and says what went" do
+    conversation = [
+      said(:system, "instructions"),
+      said(:user, "first question"),
+      said(:assistant, "a" * 4_000),
+      said(:user, "second question"),
+      said(:assistant, "b" * 4_000),
+      said(:user, "third question"),
+      said(:assistant, "c" * 400),
+      said(:assistant, "d" * 400),
+    ]
+
+    # Room for the task and one historical turn, so the oldest turn goes and
+    # the note takes its place, directly after the instructions.
+    env = compacted(conversation, 1_400, keep_steps: 1)
+    env[:conversation].map { |m| m.content[0, 24] }.should == [
+      "instructions",
+      "[compacted:sliding_windo",
+      "second question",
+      "bbbbbbbbbbbbbbbbbbbbbbbb",
+      "third question",
+      "cccccccccccccccccccccccc",
+      "dddddddddddddddddddddddd",
+    ]
+    env[:conversation][1].content.should ==
+      "[compacted:sliding_window] 2 earlier messages were dropped to free context. They cannot be recovered."
+
+    # The conversation it was handed is left as it was found.
+    conversation.length.should == 8
+
+    # A target the task cannot fit inside: every turn goes, and the task's own
+    # oldest step goes with them -- the note now sits after the anchor.
+    tight = compacted(conversation, 150, keep_steps: 1)[:conversation]
+    tight.map { |m| m.content[0, 24] }.should == [
+      "instructions",
+      "third question",
+      "[compacted:sliding_windo",
+      "dddddddddddddddddddddddd",
+    ]
+
+    # keep_steps holds the floor: the newest step survives any target at all.
+    compacted(conversation, 0, keep_steps: 1)[:conversation].length.should == 4
+
+    # Already inside the target, and a conversation that is nothing but a note.
+    compacted(conversation, 100_000, keep_steps: 1)[:applied].should == []
+    compacted(tight, 0, keep_steps: 1)[:applied].should == []
+  end
+end

data/lib/brute/compaction/middleware/strategy.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "brute"
+
+module Brute
+  module Compaction
+    # Compaction strategies, as the layers of a Brute::Turn::CompactionPipeline.
+    module Middleware
+      # One strategy in the ladder.
+      #
+      # The stack order is the policy. A layer that got the conversation under
+      # target never calls the next one, so the strategies that cost nothing
+      # sit at the top and the one that spends a model call is only reached
+      # when they could not get there.
+      #
+      # A subclass answers #strategy and #rewrite, and nothing else. The rules
+      # every strategy has to keep are here: it is only asked while the
+      # conversation is over target, and what it answers is only taken when it
+      # actually made the conversation smaller -- otherwise the pipeline above
+      # would write the same size back on every step, forever.
+      #
+      # The same class serves as a layer or as the terminal, which is what the
+      # ladder is built out of: the strategies that cost nothing are `use`d,
+      # and the one that spends a model call is `run`, reached only when they
+      # could not get under target.
+      #
+      #   Brute::Turn::CompactionPipeline.new do
+      #     use Brute::Compaction::Middleware::ToolResults
+      #     use Brute::Compaction::Middleware::SlidingWindow
+      #     run Brute::Compaction::Middleware::Summary.new(summarize: summarize)
+      #   end
+      #
+      class Strategy < Brute::Middleware::Base
+        def call(env)
+          @env = env
+
+          if Brute::Compaction.over_target?(env)
+            apply(env)
+          end
+
+          # Under target now, so the rest of the ladder is not needed.
+          if Brute::Compaction.over_target?(env)
+            @app.call(env)
+          end
+
+          env
+        end
+
+        # The name this strategy compacts under, recorded on what it produces.
+        def strategy = raise(NotImplementedError, "#{self.class} must answer #strategy")
+
+        # Answer a smaller conversation, or nil to decline. Build a new list:
+        # the one handed over belongs to the caller.
+        def rewrite(_conversation, target:) = raise(NotImplementedError, "#{self.class} must answer #rewrite")
+
+        private
+
+          # Measured with whatever counter the turn decided on, so a strategy
+          # weighs the conversation the same way the trigger did.
+          def tokens(messages) = Brute::Compaction.counter(@env).count(messages)
+
+          def apply(env)
+            rewritten = rewrite(env[:conversation], target: env[:target])
+
+            unless rewritten.nil?
+              before = tokens(env[:conversation])
+              after = tokens(rewritten)
+
+              if after < before
+                env[:conversation] = rewritten
+                env[:applied] << { strategy: strategy, before: before, after: after }
+                env[:events] << { type: :compacted, data: env[:applied].last }
+              end
+            end
+          end
+      end
+    end
+  end
+end
+
+__END__
+
+describe "brute/compaction/middleware/strategy" do
+  def said(content) = Brute::Message.new(role: :user, content: content)
+
+  def strategy(name, &block)
+    Class.new(Brute::Compaction::Middleware::Strategy) do
+      define_method(:strategy) { name }
+      define_method(:rewrite) { |conversation, target:| block.call(conversation, target) }
+    end
+  end
+
+  it "asks a strategy only while over target, takes only what shrank, and stops the ladder there" do
+    asked = []
+    halves  = strategy("halves")  { |c, _t| asked << "halves"; c.first(c.length / 2) }
+    nothing = strategy("nothing") { |c, _t| asked << "nothing"; c.dup }
+    never   = strategy("never")   { |_c, _t| asked << "never"; nil }
+
+    run = lambda do |layers, conversation, target|
+      app = ->(env) { env }
+      layers.reverse_each { |layer| app = layer.new(app) }
+      env = { conversation: conversation, target: target, applied: [], events: [] }
+      app.call(env)
+      env
+    end
+
+    four = [said("a" * 4_000), said("b" * 4_000), said("c" * 4_000), said("d" * 4_000)]
+
+    # Halving is enough, so nothing below it is reached -- which is what keeps
+    # the strategy that costs money out of a turn the free ones could handle.
+    env = run.call([halves, never], four, 3_000)
+    env[:conversation].map { |m| m.content[0, 1] }.should == ["a", "b"]
+    asked.should == ["halves"]
+    env[:applied].should == [{ strategy: "halves", before: 4_022, after: 2_011 }]
+    env[:events].should == [{ type: :compacted, data: { strategy: "halves", before: 4_022, after: 2_011 } }]
+
+    # A rewrite that saved nothing is refused and the ladder carries on past
+    # it, so the strategy itself never has to check.
+    asked.clear
+    env = run.call([nothing, halves], four, 3_000)
+    asked.should == ["nothing", "halves"]
+    env[:applied].map { |a| a[:strategy] }.should == ["halves"]
+
+    # Already under target: nothing is asked at all.
+    asked.clear
+    run.call([halves, never], four, 100_000)[:applied].should == []
+    asked.should == []
+
+    # A strategy that declines lets the next one try.
+    asked.clear
+    run.call([never, halves], four, 3_000)[:applied].map { |a| a[:strategy] }.should == ["halves"]
+
+    # The base insists a subclass says what it is and what it does.
+    bare = Class.new(Brute::Compaction::Middleware::Strategy).new(->(e) { e })
+    should.raise(NotImplementedError) { bare.strategy }
+    should.raise(NotImplementedError) { bare.rewrite([], target: 0) }
+  end
+end

data/lib/brute/compaction/middleware/tool_results.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+require "brute"
+
+module Brute
+  module Compaction
+    module Middleware
+      # The cheapest thing to give up first: what the tools said.
+      #
+      # Tool output dominates a long run, and most of it stops being useful the
+      # moment the model has acted on it. So the results are rewritten in place
+      # rather than removed -- every call keeps the result that answers it, the
+      # model can still see what it ran, and it can run it again if it turns out
+      # it still needed the answer.
+      #
+      # Oldest first, and it stops the moment the transcript is under target, so
+      # the most recent output survives the longest.
+      class ToolResults < Strategy
+        PLACEHOLDER = "Result dropped to free context. Call %s again if you still need it."
+
+        def initialize(app = nil, keep_steps: 1, min_tokens: 200)
+          if keep_steps < 1
+            raise ArgumentError, "keep_steps must be at least 1: the model has not acted on the newest results yet"
+          end
+
+          @app = app
+          @keep_steps = keep_steps
+          @min_tokens = min_tokens
+        end
+
+        def strategy = "tool_results"
+
+        def rewrite(messages, target:)
+          guarded = guarded_indices(messages)
+          names   = tool_names(messages)
+          pruned  = messages.dup
+          running = tokens(messages)
+          changed = false
+
+          messages.each_with_index do |message, index|
+            if running <= target
+              break
+            end
+
+            replacement = prune(message, names[message.tool_call_id])
+
+            unless guarded.include?(index) || replacement.nil?
+              running -= tokens([message]) - tokens([replacement])
+              pruned[index] = replacement
+              changed = true
+            end
+          end
+
+          if changed
+            pruned
+          end
+        end
+
+        private
+
+          # The newest tool-calling steps are never touched, however far over
+          # target that leaves the transcript.
+          def guarded_indices(messages)
+            Brute::Compaction::Transcript.steps(messages, from: 0)
+              .select { |span| span.count > 1 }
+              .last(@keep_steps)
+              .flat_map(&:to_a)
+          end
+
+          # A tool result carries an id, not a name -- the name is on the call it
+          # answers, so the placeholder has to go and find it.
+          def tool_names(messages)
+            messages.each_with_object({}) do |message, names|
+              message.tool_calls&.each { |call| names[call.id] = call.name }
+            end
+          end
+
+          def prune(message, name)
+            if message.role == :tool && !Brute::Compaction::Transcript.marked?(message) &&
+               tokens([message]) > @min_tokens
+              Brute::Message.new(
+                role:         :tool,
+                content:      Brute::Compaction::Transcript.mark(strategy, format(PLACEHOLDER, name || "the tool")),
+                tool_call_id: message.tool_call_id,
+              )
+            end
+          end
+      end
+    end
+  end
+end
+
+__END__
+
+describe "brute/compaction/middleware/tool_results" do
+  def call(id) = { id: id, name: "shell", arguments: { "command" => "ls" } }
+  def result(id, size) = Brute::Message.new(role: :tool, content: "x" * size, tool_call_id: id)
+  def calling(id) = Brute::Message.new(role: :assistant, content: "", tool_calls: [call(id)])
+
+  def compacted(conversation, target, **options)
+    env = { conversation: conversation, target: target, applied: [], events: [] }
+    Brute::Compaction::Middleware::ToolResults.new(->(e) { e }, **options).call(env)
+    env
+  end
+
+  it "replaces the oldest fat tool results, guarding the newest step, and stops at the target" do
+    conversation = [
+      Brute::Message.new(role: :system, content: "instructions"),
+      Brute::Message.new(role: :user, content: "go"),
+      calling("tc1"), result("tc1", 8_000),
+      calling("tc2"), result("tc2", 8_000),
+      calling("tc3"), result("tc3", 8_000),
+    ]
+
+    env = compacted(conversation, 4_500, keep_steps: 1, min_tokens: 200)
+    kept = env[:conversation]
+
+    # The oldest result went. The placeholder names the tool that produced it,
+    # and it keeps the id, so the call it answers is still answered.
+    kept[3].content.should ==
+      "[compacted:tool_results] Result dropped to free context. Call shell again if you still need it."
+    kept[3].tool_call_id.should == "tc1"
+
+    # One was enough to reach the target, so the next is untouched...
+    kept[5].content.should == "x" * 8_000
+    # ...and the newest step is guarded whatever the target says.
+    kept[7].content.should == "x" * 8_000
+
+    # The conversation it was handed is left as it was found.
+    conversation[3].content.should == "x" * 8_000
+    env[:applied].should == [{ strategy: "tool_results", before: 6_088, after: 4_111 }]
+
+    # A harsher target reaches further back, but never past the guard.
+    again = compacted(kept, 1, keep_steps: 1, min_tokens: 200)[:conversation]
+    Brute::Compaction::Transcript.marked?(again[5]).should.be.true
+    again[7].content.should == "x" * 8_000
+
+    # Nothing left to give up, already under target, or nothing fat enough.
+    compacted(again, 1, keep_steps: 1, min_tokens: 200)[:applied].should == []
+    compacted(conversation, 100_000, keep_steps: 1)[:applied].should == []
+    compacted(conversation, 1, keep_steps: 1, min_tokens: 100_000)[:applied].should == []
+
+    should.raise(ArgumentError) { Brute::Compaction::Middleware::ToolResults.new(->(e) { e }, keep_steps: 0) }
+  end
+end