brute 5.0.4 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/brute/compaction/middleware/sliding_window.rb +170 -0
- data/lib/brute/compaction/middleware/strategy.rb +139 -0
- data/lib/brute/compaction/middleware/tool_results.rb +146 -0
- data/lib/brute/compaction/summarize.rb +340 -0
- data/lib/brute/compaction/transcript.rb +166 -0
- data/lib/brute/compaction.rb +57 -0
- data/lib/brute/contrib/otel.rb +2 -2
- data/lib/brute/env.rb +54 -0
- data/lib/brute/eval/case.rb +249 -0
- data/lib/brute/eval/suite.rb +176 -0
- data/lib/brute/eval/transcript.rb +147 -0
- data/lib/brute/eval/world.rb +106 -0
- data/lib/brute/eval.rb +48 -0
- data/lib/brute/hooks.rb +11 -0
- data/lib/brute/middleware/000_base.rb +1 -1
- data/lib/brute/middleware/001_slash_commands.rb +88 -0
- data/lib/brute/middleware/008_checkpoint.rb +1 -1
- data/lib/brute/middleware/040_compaction_check.rb +51 -135
- data/lib/brute/middleware/040_default_compaction_pipeline.rb +352 -0
- data/lib/brute/middleware/070_default_tool_pipeline.rb +327 -0
- data/lib/brute/middleware/070_tool_pipeline.rb +39 -305
- data/lib/brute/token_counter/approximate.rb +54 -0
- data/lib/brute/token_counter/tiktoken.rb +80 -0
- data/lib/brute/token_counter.rb +148 -0
- data/lib/brute/tools/adapter.rb +4 -4
- data/lib/brute/tools/skill_load.rb +2 -2
- data/lib/brute/tools/sub_agent.rb +2 -2
- data/lib/brute/turn/agent_pipeline.rb +117 -38
- data/lib/brute/turn/compaction_pipeline.rb +122 -0
- data/lib/brute/version.rb +1 -1
- data/lib/brute.rb +16 -1
- metadata +20 -1
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "bundler/setup"
|
|
4
|
+
require "brute"
|
|
5
|
+
|
|
6
|
+
module Brute
|
|
7
|
+
module Middleware
|
|
8
|
+
# Compacts the conversation once it fills too much of the model's window.
|
|
9
|
+
#
|
|
10
|
+
# This layer owns the *when*: before each call it estimates the size of the
|
|
11
|
+
# conversation, and once it reaches `compact_at` of the window it asks a
|
|
12
|
+
# compactor to bring it down to `compact_to`. What is given up, and in what
|
|
13
|
+
# order, is the compactor's business.
|
|
14
|
+
#
|
|
15
|
+
# use Brute::Middleware::Loop::ToolResult
|
|
16
|
+
# use Brute::Middleware::DefaultCompactionPipeline,
|
|
17
|
+
# window: 200_000,
|
|
18
|
+
# summariser: Brute::Completion::OpenRouter.new(config: { access_token: key })
|
|
19
|
+
# use Brute::Middleware::DefaultToolPipeline, tools: tools
|
|
20
|
+
#
|
|
21
|
+
# The compactor it builds is the usual ladder, cheapest first:
|
|
22
|
+
#
|
|
23
|
+
# use Brute::Compaction::Middleware::ToolResults rewrite older tool output
|
|
24
|
+
# use Brute::Compaction::Middleware::SlidingWindow drop the oldest turns, then steps
|
|
25
|
+
# run Brute::Compaction::Summarize summarise what is left, until it fits
|
|
26
|
+
#
|
|
27
|
+
# Pass `compactor:` to say something else. A Brute::Turn::CompactionPipeline
|
|
28
|
+
# is the usual thing to pass, but anything answering
|
|
29
|
+
# `#compact(messages, target:)` will do. Passing no summariser leaves the
|
|
30
|
+
# terminal doing nothing, which is how an agent says it would rather live
|
|
31
|
+
# with a full context than pay to shrink it.
|
|
32
|
+
#
|
|
33
|
+
# It belongs inside the tool loop rather than around it, so it runs before
|
|
34
|
+
# every call rather than once a turn -- a long run of tool results can fill
|
|
35
|
+
# the window without the turn ever ending.
|
|
36
|
+
#
|
|
37
|
+
# Sizing anchors on what the provider itself counted for the last call and
|
|
38
|
+
# measures locally only what has been appended since, so the estimate is
|
|
39
|
+
# exact for the bulk of the conversation and approximate only for its tail.
|
|
40
|
+
#
|
|
41
|
+
# Tool schemas ride in every request and are part of what fills a window,
|
|
42
|
+
# so they are counted too -- against the trigger when nothing has been
|
|
43
|
+
# reported yet, and off the target the compactor is given, because what
|
|
44
|
+
# the schemas occupy is not room the conversation may have. This layer
|
|
45
|
+
# sits above the tool pipeline, so on the very first call of a turn
|
|
46
|
+
# env[:tools] is not set yet: pass `tools:` to have them counted then.
|
|
47
|
+
#
|
|
48
|
+
# Compaction is lossy, and this layer keeps no record of what it gave up:
|
|
49
|
+
# it rewrites env[:messages] and says so with :compacted. An application
|
|
50
|
+
# that keeps a transcript preserves it from there.
|
|
51
|
+
#
|
|
52
|
+
# agent.on(:compacted) { |env, payload| archive(env, payload) }
|
|
53
|
+
#
|
|
54
|
+
class DefaultCompactionPipeline < Brute::Middleware::Base
|
|
55
|
+
def initialize(app, window:, summariser: nil, compactor: nil, keep_steps: 2,
|
|
56
|
+
compact_at: 0.7, compact_to: 0.4, token_counter: nil, tools: nil)
|
|
57
|
+
unless compact_to.positive? && compact_to < compact_at && compact_at <= 1
|
|
58
|
+
raise ArgumentError, "expected 0 < compact_to < compact_at <= 1, got #{compact_to} and #{compact_at}"
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
@app = app
|
|
62
|
+
@compactor = compactor || self.class.compactor(summariser: summariser, keep_steps: keep_steps)
|
|
63
|
+
@window = window
|
|
64
|
+
@compact_at = compact_at
|
|
65
|
+
@compact_to = compact_to
|
|
66
|
+
@token_counter = token_counter || Brute::TokenCounter.default
|
|
67
|
+
@tools = tools
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def call(env)
|
|
71
|
+
env[:compactor] = @compactor
|
|
72
|
+
compact(env)
|
|
73
|
+
@app.call(env)
|
|
74
|
+
env
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# The ladder this layer wires when it is not given one.
|
|
78
|
+
def self.compactor(summariser: nil, keep_steps: 2)
|
|
79
|
+
terminal = Declines.new
|
|
80
|
+
|
|
81
|
+
unless summariser.nil?
|
|
82
|
+
terminal = Brute::Compaction::Summarize.new(summariser, keep_steps: keep_steps)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
Brute::Turn::CompactionPipeline.new do
|
|
86
|
+
use Brute::Compaction::Middleware::ToolResults, keep_steps: 1
|
|
87
|
+
use Brute::Compaction::Middleware::SlidingWindow, keep_steps: keep_steps
|
|
88
|
+
|
|
89
|
+
run terminal
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# The floor of a pipeline given nothing to summarise with: what the free
|
|
94
|
+
# layers managed is what the turn gets.
|
|
95
|
+
class Declines
|
|
96
|
+
def call(env) = env
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
private
|
|
100
|
+
|
|
101
|
+
def compact(env)
|
|
102
|
+
context = estimate(env)
|
|
103
|
+
|
|
104
|
+
if context >= @window * @compact_at
|
|
105
|
+
before = @token_counter.count(env[:messages])
|
|
106
|
+
compacted = attempt(env, target(env))
|
|
107
|
+
|
|
108
|
+
unless compacted.nil?
|
|
109
|
+
env[:messages].replace(compacted)
|
|
110
|
+
# The reported total describes a conversation that no longer
|
|
111
|
+
# exists, so the next estimate must not build on it. The call
|
|
112
|
+
# below refreshes it; one that fails leaves the estimate to be
|
|
113
|
+
# counted from scratch instead of anchored to a fiction.
|
|
114
|
+
env[:metadata]&.delete(:last_llm_usage)
|
|
115
|
+
emit(
|
|
116
|
+
COMPACTED_EVENT,
|
|
117
|
+
env,
|
|
118
|
+
{ context: context, before: before, after: @token_counter.count(compacted) },
|
|
119
|
+
)
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# What the conversation alone may occupy: the target, less whatever
|
|
125
|
+
# the schemas are already taking out of it. A window too small to
|
|
126
|
+
# hold its own tool schemas is a configuration a compactor cannot
|
|
127
|
+
# fix, so the floor is 1 rather than a negative target.
|
|
128
|
+
def target(env)
|
|
129
|
+
[(@window * @compact_to).to_i - schemas(env), 1].max
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def schemas(env)
|
|
133
|
+
tools = @tools || env[:tools]
|
|
134
|
+
|
|
135
|
+
if tools.nil?
|
|
136
|
+
0
|
|
137
|
+
else
|
|
138
|
+
@token_counter.count([], tools: tools)
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
# A compactor that raises is a compactor that declined. Compaction is
|
|
143
|
+
# an optimisation, and an agent that cannot shrink its context should
|
|
144
|
+
# carry on with the context it has rather than die of it.
|
|
145
|
+
def attempt(env, target)
|
|
146
|
+
compacted = nil
|
|
147
|
+
|
|
148
|
+
emit(COMPACT_DURATION_EVENT, env, env[:compactor]) do
|
|
149
|
+
compacted = env[:compactor].compact(
|
|
150
|
+
env[:messages],
|
|
151
|
+
target: target,
|
|
152
|
+
events: env[:events] || Brute::Turn::Pipeline::NullSink.new,
|
|
153
|
+
token_counter: @token_counter,
|
|
154
|
+
)
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
compacted
|
|
158
|
+
rescue => error
|
|
159
|
+
(env[:events] ||= []) << { type: :error, data: { error: error, message: error.message } }
|
|
160
|
+
nil
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# What the provider counted for the last call already covers the
|
|
164
|
+
# system prompt, the tool schemas and the chat-template overhead, so
|
|
165
|
+
# only what has landed since it answered needs counting here.
|
|
166
|
+
def estimate(env)
|
|
167
|
+
Brute::TokenCounter.estimate(env, counter: @token_counter, tools: @tools || env[:tools])
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
__END__
|
|
174
|
+
|
|
175
|
+
describe "brute/middleware/040_default_compaction_pipeline" do
|
|
176
|
+
def conversation
|
|
177
|
+
Brute.log.tap do |log|
|
|
178
|
+
log.user("ask" + ("x" * 1_000))
|
|
179
|
+
log.assistant("answer" + ("y" * 1_000))
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def usage(total) = { last_llm_usage: Brute::UsageDetection::Usage.new(total: total) }
|
|
184
|
+
|
|
185
|
+
def history(turns: 4, size: 6_000)
|
|
186
|
+
Brute.log.tap do |log|
|
|
187
|
+
log.system("instructions")
|
|
188
|
+
turns.times do |n|
|
|
189
|
+
log.user("question #{n}")
|
|
190
|
+
log << Brute::Message.new(role: :assistant, content: n.to_s * size)
|
|
191
|
+
end
|
|
192
|
+
log.user("the current task")
|
|
193
|
+
log << Brute::Message.new(role: :assistant, content: "z" * size)
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def compactor(&block)
|
|
198
|
+
Object.new.tap do |double|
|
|
199
|
+
double.define_singleton_method(:compact) { |messages, target:, **| block.call(messages, target) }
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
it "compacts only once the window fills, and says what the turn gave up" do
|
|
204
|
+
asked = []
|
|
205
|
+
events = []
|
|
206
|
+
shrinks = compactor { |messages, target| asked << target; messages.first(1) }
|
|
207
|
+
declines = compactor { |_messages, target| asked << target; nil }
|
|
208
|
+
raises = compactor { |_messages, _target| raise IOError, "the summariser is down" }
|
|
209
|
+
|
|
210
|
+
layer = lambda do |compactor|
|
|
211
|
+
Brute::Middleware::DefaultCompactionPipeline.new(
|
|
212
|
+
->(env) { env },
|
|
213
|
+
compactor: compactor,
|
|
214
|
+
window: 1_000,
|
|
215
|
+
compact_at: 0.7,
|
|
216
|
+
compact_to: 0.4,
|
|
217
|
+
).tap do |it|
|
|
218
|
+
it.define_singleton_method(:emit) { |event, _env, *extras, &work| work&.call; events << [event, *extras] }
|
|
219
|
+
end
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
env = { messages: conversation, metadata: usage(900) }
|
|
223
|
+
layer.call(shrinks).call(env)
|
|
224
|
+
|
|
225
|
+
asked.should == [400]
|
|
226
|
+
env[:messages].map(&:role).should == [:user]
|
|
227
|
+
env[:compactor].equal?(shrinks).should.be.true
|
|
228
|
+
events.select { |e, _| e == :compacted }.should == [[:compacted, { context: 900, before: 514, after: 256 }]]
|
|
229
|
+
# The attempt is timed, so the compactor that spends a model call is visible.
|
|
230
|
+
events.select { |e, _| e == :compact_duration }.length.should == 1
|
|
231
|
+
|
|
232
|
+
# Sizing anchors on what the provider counted: under compact_at nothing is
|
|
233
|
+
# asked and nothing is said.
|
|
234
|
+
asked.clear
|
|
235
|
+
events.clear
|
|
236
|
+
layer.call(shrinks).call({ messages: conversation, metadata: usage(600) })
|
|
237
|
+
asked.should == []
|
|
238
|
+
events.should == []
|
|
239
|
+
|
|
240
|
+
# ...but what landed since it answered is counted on top of it, and that
|
|
241
|
+
# is what tips this one over.
|
|
242
|
+
tail = conversation.tap { |log| log.tool("z" * 400, tool_call_id: "tc1") }
|
|
243
|
+
layer.call(declines).call({ messages: tail, metadata: usage(600) })
|
|
244
|
+
asked.should == [400]
|
|
245
|
+
|
|
246
|
+
# With nothing reported at all, the whole conversation is counted here.
|
|
247
|
+
asked.clear
|
|
248
|
+
layer.call(declines).call({ messages: conversation })
|
|
249
|
+
asked.should == []
|
|
250
|
+
|
|
251
|
+
# A compactor that raises is a compactor that declined: reported, not
|
|
252
|
+
# fatal. An agent that cannot shrink its context carries on with it.
|
|
253
|
+
failing = { messages: conversation, events: [], metadata: usage(900) }
|
|
254
|
+
should.not.raise(IOError) { layer.call(raises).call(failing) }
|
|
255
|
+
failing[:messages].length.should == 2
|
|
256
|
+
failing[:events].map { |e| [e[:type], e[:data][:message]] }.should == [[:error, "the summariser is down"]]
|
|
257
|
+
|
|
258
|
+
# A target at or above the trigger would compact on every single step.
|
|
259
|
+
should.raise(ArgumentError) do
|
|
260
|
+
Brute::Middleware::DefaultCompactionPipeline.new(->(e) { e }, compactor: shrinks, window: 10,
|
|
261
|
+
compact_at: 0.4, compact_to: 0.7)
|
|
262
|
+
end
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
it "wires the usual ladder, cheapest first, and only pays when the free layers cannot" do
|
|
266
|
+
rounds = 0
|
|
267
|
+
summariser = lambda do |env|
|
|
268
|
+
rounds += 1
|
|
269
|
+
env[:messages] << Brute::Message.new(role: :assistant, content: "what happened")
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
ladder = Brute::Middleware::DefaultCompactionPipeline.compactor(summariser: summariser, keep_steps: 1)
|
|
273
|
+
ladder.should.be.kind_of Brute::Turn::CompactionPipeline
|
|
274
|
+
|
|
275
|
+
events = []
|
|
276
|
+
sink = Object.new.tap { |it| it.define_singleton_method(:<<) { |e| events << e; it } }
|
|
277
|
+
|
|
278
|
+
messages = history
|
|
279
|
+
before = Brute::Compaction::Transcript.tokens(messages)
|
|
280
|
+
out = ladder.compact(messages, target: 1_500, events: sink)
|
|
281
|
+
|
|
282
|
+
Brute::Compaction::Transcript.tokens(out).should.be < before
|
|
283
|
+
events.select { |e| e[:type] == :compacted }.map { |e| e[:data][:strategy] }
|
|
284
|
+
.should.include "sliding_window"
|
|
285
|
+
|
|
286
|
+
# Given no summariser the terminal declines, and the free layers are all
|
|
287
|
+
# the agent gets -- a policy said out loud rather than configured.
|
|
288
|
+
free = Brute::Middleware::DefaultCompactionPipeline.compactor(summariser: nil, keep_steps: 1)
|
|
289
|
+
free.compact(history, target: 1_500).should.not.be.nil
|
|
290
|
+
|
|
291
|
+
# One layer, so it still owns the *when* as well as the what.
|
|
292
|
+
layer = Brute::Middleware::DefaultCompactionPipeline.new(->(e) { e }, window: 1_000, summariser: summariser)
|
|
293
|
+
layer.define_singleton_method(:emit) { |_e, _env, *, &work| work&.call }
|
|
294
|
+
env = { messages: history, metadata: {} }
|
|
295
|
+
layer.call(env)
|
|
296
|
+
env[:messages].length.should.be < 11
|
|
297
|
+
end
|
|
298
|
+
|
|
299
|
+
it "counts the schemas it will send, and stops anchoring on a conversation that is gone" do
|
|
300
|
+
asked = []
|
|
301
|
+
shrinks = compactor { |messages, target| asked << target; messages.first(1) }
|
|
302
|
+
|
|
303
|
+
tool = Brute::Turn::ToolPipeline.new(name: "search", description: "search the web. " * 40) do
|
|
304
|
+
run ->(env) { env[:result] = "" }
|
|
305
|
+
end
|
|
306
|
+
schemas = Brute::TokenCounter.default.count([], tools: [tool])
|
|
307
|
+
|
|
308
|
+
layer = lambda do |**options|
|
|
309
|
+
Brute::Middleware::DefaultCompactionPipeline.new(
|
|
310
|
+
->(env) { env },
|
|
311
|
+
compactor: shrinks,
|
|
312
|
+
window: 1_000,
|
|
313
|
+
**options,
|
|
314
|
+
).tap { |it| it.define_singleton_method(:emit) { |_event, _env, *, &work| work&.call } }
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# 650 tokens of conversation, nothing reported yet: under the 700 the
|
|
318
|
+
# window triggers at, so on its own it is left alone.
|
|
319
|
+
said = -> { Brute.log.tap { |log| log.user("x" * 2_578) } }
|
|
320
|
+
Brute::TokenCounter.default.count(said.call).should == 650
|
|
321
|
+
|
|
322
|
+
layer.call.call({ messages: said.call, metadata: {} })
|
|
323
|
+
asked.should == []
|
|
324
|
+
|
|
325
|
+
# The same conversation ships with tool schemas in every request, and
|
|
326
|
+
# counting what they occupy is what tips it over. What is left of the
|
|
327
|
+
# target after them is what the compactor is asked for -- the schemas are
|
|
328
|
+
# not room the conversation may have.
|
|
329
|
+
layer.call(tools: [tool]).call({ messages: said.call, metadata: {} })
|
|
330
|
+
asked.should == [400 - schemas]
|
|
331
|
+
|
|
332
|
+
# Reported usage already covers the schemas, so the warm path must not
|
|
333
|
+
# count them again -- 600 reported and nothing said since is 600, tools
|
|
334
|
+
# or no tools.
|
|
335
|
+
asked.clear
|
|
336
|
+
layer.call(tools: [tool]).call({ messages: conversation, metadata: usage(600) })
|
|
337
|
+
asked.should == []
|
|
338
|
+
|
|
339
|
+
# What the provider counted describes the conversation as it was. Once
|
|
340
|
+
# compaction has given part of it up that number is a fiction, so it goes
|
|
341
|
+
# -- the next estimate counts from scratch rather than building on it.
|
|
342
|
+
compacted = { messages: conversation, metadata: usage(900) }
|
|
343
|
+
layer.call.call(compacted)
|
|
344
|
+
compacted[:messages].length.should == 1
|
|
345
|
+
compacted[:metadata].key?(:last_llm_usage).should.be.false
|
|
346
|
+
|
|
347
|
+
# Nothing given up, nothing invalidated.
|
|
348
|
+
kept = { messages: conversation, metadata: usage(900) }
|
|
349
|
+
layer.call(compactor: compactor { |_messages, _target| nil }).call(kept)
|
|
350
|
+
kept[:metadata].key?(:last_llm_usage).should.be.true
|
|
351
|
+
end
|
|
352
|
+
end
|
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "bundler/setup"
|
|
4
|
+
require "brute"
|
|
5
|
+
require "brute/truncation"
|
|
6
|
+
require "async"
|
|
7
|
+
require "async/barrier"
|
|
8
|
+
|
|
9
|
+
module Brute
|
|
10
|
+
module Middleware
|
|
11
|
+
class DefaultToolPipeline < Brute::Middleware::Base
|
|
12
|
+
def initialize(app, tools: [])
|
|
13
|
+
@app = app
|
|
14
|
+
@tools = tools
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def call(env)
|
|
18
|
+
env[:tools] = @tools
|
|
19
|
+
@app.call(env)
|
|
20
|
+
|
|
21
|
+
response = env[:messages].last
|
|
22
|
+
|
|
23
|
+
if response.respond_to?(:tool_calls) && response.tool_calls.present?
|
|
24
|
+
tools_to_run = response.tool_calls
|
|
25
|
+
|
|
26
|
+
# tool_to_run may be an Array (Brute::ToolCall) or an id-keyed Hash (some libraries' native shape)
|
|
27
|
+
if tools_to_run.respond_to?(:values)
|
|
28
|
+
tools_to_run = tools_to_run.values
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# No idea why we use Array() here... probably in case it's a hash or something...
|
|
32
|
+
# because reject would work on the hash...
|
|
33
|
+
tools_to_run = Array(tools_to_run).reject { |tc| tc.name == "question" }
|
|
34
|
+
|
|
35
|
+
available_tools = Brute::Tools::Adapter.wrap_all(env[:tools])
|
|
36
|
+
env[:events] << on_tool_call_start_event(tools_to_run)
|
|
37
|
+
|
|
38
|
+
results = []
|
|
39
|
+
|
|
40
|
+
# Async::Barrier blocks until all tasks are complete.
|
|
41
|
+
# Tasks run in parrallel.
|
|
42
|
+
#
|
|
43
|
+
Sync do
|
|
44
|
+
barrier = Async::Barrier.new
|
|
45
|
+
|
|
46
|
+
tools_to_run.each do |tool_call|
|
|
47
|
+
barrier.async do
|
|
48
|
+
name = tool_call.name.to_sym
|
|
49
|
+
args = tool_call.arguments
|
|
50
|
+
|
|
51
|
+
# Lifecycle hooks (Brute::Hooks): before_tool may rewrite
|
|
52
|
+
# :arguments or short-circuit with a :result; approve_tool
|
|
53
|
+
# denies on a false (or String) return; after_tool may
|
|
54
|
+
# rewrite :result.
|
|
55
|
+
call_env = {
|
|
56
|
+
name: name.to_s,
|
|
57
|
+
arguments: args,
|
|
58
|
+
result: nil,
|
|
59
|
+
denied: nil,
|
|
60
|
+
events: env[:events],
|
|
61
|
+
metadata: {},
|
|
62
|
+
turn_env: env,
|
|
63
|
+
}
|
|
64
|
+
# A subscriber takes part by mutating the call env: set
|
|
65
|
+
# :result to answer without executing, set :denied to refuse.
|
|
66
|
+
emit(BEFORE_TOOL_EVENT, env, call_env)
|
|
67
|
+
|
|
68
|
+
if call_env[:result].nil?
|
|
69
|
+
emit(APPROVE_TOOL_EVENT, env, call_env)
|
|
70
|
+
|
|
71
|
+
if (denial = call_env[:denied])
|
|
72
|
+
call_env[:result] = denial.is_a?(String) ? denial : %(Tool call to "#{name}" was denied.)
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# Only the tool's own execution is timed: a call that
|
|
77
|
+
# before_tool answered, or approve_tool denied, never ran.
|
|
78
|
+
result = call_env[:result]
|
|
79
|
+
|
|
80
|
+
if result.nil?
|
|
81
|
+
emit(TOOL_DURATION_EVENT, env, call_env) do
|
|
82
|
+
result = available_tools[name].call(call_env[:arguments])
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
call_env[:result] = result
|
|
87
|
+
emit(AFTER_TOOL_EVENT, env, call_env)
|
|
88
|
+
result = call_env[:result]
|
|
89
|
+
|
|
90
|
+
# Coerce to String so Hash results (e.g. Shell's
|
|
91
|
+
# {stdout:, stderr:, exit_code:}) serialize predictably.
|
|
92
|
+
if result.is_a?(String)
|
|
93
|
+
content = result
|
|
94
|
+
else
|
|
95
|
+
content = result.to_s
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Universal truncation safety net — skip if already truncated
|
|
99
|
+
unless Brute::Truncation.already_truncated?(content)
|
|
100
|
+
content = Brute::Truncation.truncate(content)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
results << [tool_call, content]
|
|
104
|
+
rescue => e
|
|
105
|
+
# Capture the error as a tool result so the LLM can see it
|
|
106
|
+
# and reason about the failure, rather than crashing the
|
|
107
|
+
# entire middleware chain.
|
|
108
|
+
env[:events] << { type: :error, data: { error: e, message: e.message } }
|
|
109
|
+
results << [tool_call, "Error: #{e.class}: #{e.message}"]
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
barrier.wait
|
|
114
|
+
ensure
|
|
115
|
+
barrier&.cancel
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
# Append events and messages in the original tool_call order so the
|
|
119
|
+
# LLM sees a deterministic sequence regardless of completion order.
|
|
120
|
+
results.sort_by! { |tool_call, _| tools_to_run.index(tool_call) }
|
|
121
|
+
|
|
122
|
+
results.each do |tool_call, content|
|
|
123
|
+
env[:events] << { type: :tool_result, data: { name: tool_call.name, content: content } }
|
|
124
|
+
env[:messages] << Brute::Message.new(role: :tool, content: content, tool_call_id: tool_call.id)
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
env
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
private
|
|
132
|
+
|
|
133
|
+
def on_tool_call_start_event(pending_tools)
|
|
134
|
+
{
|
|
135
|
+
type: :tool_call_start,
|
|
136
|
+
data: pending_tools.map { |tc|
|
|
137
|
+
{
|
|
138
|
+
name: tc.name,
|
|
139
|
+
call_id: tc.id,
|
|
140
|
+
arguments: tc.arguments
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
__END__
|
|
150
|
+
|
|
151
|
+
describe "brute/middleware/070_default_tool_pipeline" do
|
|
152
|
+
require "brute/messages"
|
|
153
|
+
require "brute/truncation"
|
|
154
|
+
|
|
155
|
+
it "passes through when no tool calls pending" do
|
|
156
|
+
inner = ->(env) {
|
|
157
|
+
env[:messages] << Brute::Message.new(role: :assistant, content: "hi")
|
|
158
|
+
}
|
|
159
|
+
mw = Brute::Middleware::DefaultToolPipeline.new(inner, tools: [])
|
|
160
|
+
env = {
|
|
161
|
+
messages: Brute.log,
|
|
162
|
+
events: [],
|
|
163
|
+
}
|
|
164
|
+
env[:messages].user("hello")
|
|
165
|
+
mw.call(env)
|
|
166
|
+
env[:messages].last.content.should == "hi"
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
it "advertises its tools on env[:tools] on the way in" do
|
|
170
|
+
seen = nil
|
|
171
|
+
inner = ->(env) { seen = env[:tools] }
|
|
172
|
+
tool = { name: "echo", description: "", execute: ->(**) { "ok" } }
|
|
173
|
+
mw = Brute::Middleware::DefaultToolPipeline.new(inner, tools: [tool])
|
|
174
|
+
env = { messages: Brute.log, events: [] }
|
|
175
|
+
env[:messages].user("hi")
|
|
176
|
+
mw.call(env)
|
|
177
|
+
seen.should == [tool]
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
# --- lifecycle hooks (Brute::Hooks) ---
|
|
181
|
+
|
|
182
|
+
# A layer only gets its emit from the builder that made it, so a hook spec
|
|
183
|
+
# builds a real pipeline rather than instantiating the middleware alone.
|
|
184
|
+
def hooked(inner, tools:, &subscribe)
|
|
185
|
+
pipeline = Brute::Turn::Pipeline.new
|
|
186
|
+
pipeline.use Brute::Middleware::DefaultToolPipeline, tools: tools
|
|
187
|
+
pipeline.run(Object.new.tap { |o| o.define_singleton_method(:call, &inner) })
|
|
188
|
+
subscribe.call(pipeline)
|
|
189
|
+
pipeline
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def hook_env
|
|
193
|
+
{ messages: Brute.log, events: [] }
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
it "before_tool may rewrite arguments and short-circuit with a result" do
|
|
197
|
+
tool = { name: "echo", description: "", execute: ->(text:) { "ran:#{text}" } }
|
|
198
|
+
inner = ->(env) do
|
|
199
|
+
env[:messages] << Brute::Message.new(role: :assistant, content: "",
|
|
200
|
+
tool_calls: [{ id: "tc1", name: "echo", arguments: { "text" => "orig" } }])
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
pipeline = hooked(inner, tools: [tool]) do |p|
|
|
204
|
+
p.on(:before_tool) { |_env, call| call[:arguments] = { text: "rewritten" } }
|
|
205
|
+
end
|
|
206
|
+
env = hook_env
|
|
207
|
+
env[:messages].user("hi")
|
|
208
|
+
pipeline.call(env)
|
|
209
|
+
env[:messages].last.content.should == "ran:rewritten"
|
|
210
|
+
|
|
211
|
+
canned = hooked(inner, tools: [tool]) { |p| p.on(:before_tool) { |_env, call| call[:result] = "canned" } }
|
|
212
|
+
env2 = hook_env
|
|
213
|
+
env2[:messages].user("hi")
|
|
214
|
+
canned.call(env2)
|
|
215
|
+
env2[:messages].last.content.should == "canned" # never executed
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
it "approve_tool denies on false (generic message) or String (custom)" do
|
|
219
|
+
tool = { name: "exec", description: "", execute: ->(**) { "ran" } }
|
|
220
|
+
inner = ->(env) do
|
|
221
|
+
env[:messages] << Brute::Message.new(role: :assistant, content: "",
|
|
222
|
+
tool_calls: [{ id: "tc1", name: "exec", arguments: {} }])
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
denied = hooked(inner, tools: [tool]) { |p| p.on(:approve_tool) { |_env, call| call[:denied] = true } }
|
|
226
|
+
env = hook_env
|
|
227
|
+
env[:messages].user("hi")
|
|
228
|
+
denied.call(env)
|
|
229
|
+
env[:messages].last.content.should == %(Tool call to "exec" was denied.)
|
|
230
|
+
|
|
231
|
+
by_policy = hooked(inner, tools: [tool]) { |p| p.on(:approve_tool) { |_env, call| call[:denied] = "denied by policy" } }
|
|
232
|
+
env2 = hook_env
|
|
233
|
+
env2[:messages].user("hi")
|
|
234
|
+
by_policy.call(env2)
|
|
235
|
+
env2[:messages].last.content.should == "denied by policy"
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
it "after_tool may rewrite the result" do
|
|
239
|
+
tool = { name: "echo", description: "", execute: ->(**) { "raw" } }
|
|
240
|
+
inner = ->(env) do
|
|
241
|
+
env[:messages] << Brute::Message.new(role: :assistant, content: "",
|
|
242
|
+
tool_calls: [{ id: "tc1", name: "echo", arguments: {} }])
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
pipeline = hooked(inner, tools: [tool]) do |p|
|
|
246
|
+
p.on(:after_tool) { |_env, call| call[:result] = "rewrote(#{call[:result]})" }
|
|
247
|
+
end
|
|
248
|
+
env = hook_env
|
|
249
|
+
env[:messages].user("hi")
|
|
250
|
+
pipeline.call(env)
|
|
251
|
+
env[:messages].last.content.should == "rewrote(raw)"
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
# --- Universal output truncation ---
|
|
255
|
+
|
|
256
|
+
it "truncates large tool results via Truncation" do
|
|
257
|
+
# A fake tool that returns a huge string
|
|
258
|
+
big_tool = Class.new(Brute::Tool) do
|
|
259
|
+
description "test tool"
|
|
260
|
+
param :input, type: "string", desc: "input"
|
|
261
|
+
def name; "big_tool"; end
|
|
262
|
+
def execute(input:)
|
|
263
|
+
"line\n" * 3000
|
|
264
|
+
end
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
tool_calls = [
|
|
268
|
+
Brute::ToolCall.new(
|
|
269
|
+
id: "tc_1",
|
|
270
|
+
name: "big_tool",
|
|
271
|
+
arguments: { "input" => "go" },
|
|
272
|
+
)
|
|
273
|
+
]
|
|
274
|
+
|
|
275
|
+
inner = ->(env) {
|
|
276
|
+
env[:messages] << Brute::Message.new(role: :assistant, content: "", tool_calls: tool_calls)
|
|
277
|
+
}
|
|
278
|
+
pipeline = hooked(inner, tools: [big_tool]) { |_p| nil }
|
|
279
|
+
env = {
|
|
280
|
+
messages: Brute.log,
|
|
281
|
+
events: [],
|
|
282
|
+
}
|
|
283
|
+
env[:messages].user("hello")
|
|
284
|
+
pipeline.call(env)
|
|
285
|
+
|
|
286
|
+
tool_msg = env[:messages].select { |m| m.role == :tool }.last
|
|
287
|
+
tool_msg.content.lines.size.should.be < 2100
|
|
288
|
+
tool_msg.content.should =~ /truncated/i
|
|
289
|
+
end
|
|
290
|
+
|
|
291
|
+
# --- Skip double-truncation ---
|
|
292
|
+
|
|
293
|
+
it "does not double-truncate already-truncated output" do
|
|
294
|
+
# A fake tool that returns output already containing the truncation marker
|
|
295
|
+
pre_truncated_tool = Class.new(Brute::Tool) do
|
|
296
|
+
description "test tool"
|
|
297
|
+
param :input, type: "string", desc: "input"
|
|
298
|
+
def name; "pre_truncated_tool"; end
|
|
299
|
+
def execute(input:)
|
|
300
|
+
"some result\n[Output truncated: showing 100 of 5000 lines]"
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
tool_calls = [
|
|
305
|
+
Brute::ToolCall.new(
|
|
306
|
+
id: "tc_2",
|
|
307
|
+
name: "pre_truncated_tool",
|
|
308
|
+
arguments: { "input" => "go" },
|
|
309
|
+
)
|
|
310
|
+
]
|
|
311
|
+
|
|
312
|
+
inner = ->(env) {
|
|
313
|
+
env[:messages] << Brute::Message.new(role: :assistant, content: "", tool_calls: tool_calls)
|
|
314
|
+
}
|
|
315
|
+
pipeline = hooked(inner, tools: [pre_truncated_tool]) { |_p| nil }
|
|
316
|
+
env = {
|
|
317
|
+
messages: Brute.log,
|
|
318
|
+
events: [],
|
|
319
|
+
}
|
|
320
|
+
env[:messages].user("hello")
|
|
321
|
+
pipeline.call(env)
|
|
322
|
+
|
|
323
|
+
tool_msg = env[:messages].select { |m| m.role == :tool }.last
|
|
324
|
+
# Should contain exactly one truncation marker, not two
|
|
325
|
+
tool_msg.content.scan(/Output truncated/).size.should == 1
|
|
326
|
+
end
|
|
327
|
+
end
|