brute 5.0.5 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/brute/compaction/middleware/sliding_window.rb +170 -0
- data/lib/brute/compaction/middleware/strategy.rb +137 -0
- data/lib/brute/compaction/middleware/tool_results.rb +146 -0
- data/lib/brute/compaction/summarize.rb +349 -0
- data/lib/brute/compaction/transcript.rb +166 -0
- data/lib/brute/compaction.rb +57 -0
- data/lib/brute/completion/lang_chain.rb +34 -31
- data/lib/brute/completion/llmrb.rb +31 -29
- data/lib/brute/completion/open_router.rb +43 -40
- data/lib/brute/completion/ruby_llm.rb +28 -27
- data/lib/brute/contrib/otel.rb +83 -51
- data/lib/brute/env.rb +54 -0
- data/lib/brute/eval/case.rb +254 -0
- data/lib/brute/eval/suite.rb +181 -0
- data/lib/brute/eval/transcript.rb +147 -0
- data/lib/brute/eval/world.rb +106 -0
- data/lib/brute/eval.rb +48 -0
- data/lib/brute/events/handler.rb +3 -1
- data/lib/brute/events/prefixed_terminal_output.rb +3 -1
- data/lib/brute/events/terminal_output_handler.rb +1 -1
- data/lib/brute/hooks.rb +89 -102
- data/lib/brute/message_transport/anthropic.rb +13 -3
- data/lib/brute/message_transport/llm.rb +8 -4
- data/lib/brute/message_transport/open_router.rb +30 -16
- data/lib/brute/message_transport/openai.rb +14 -7
- data/lib/brute/message_transport/ruby_llm.rb +21 -17
- data/lib/brute/message_transport/ruby_open_ai.rb +59 -57
- data/lib/brute/messages.rb +15 -8
- data/lib/brute/middleware/000_base.rb +6 -6
- data/lib/brute/middleware/002_session_log.rb +12 -4
- data/lib/brute/middleware/008_checkpoint.rb +27 -19
- data/lib/brute/middleware/010_max_iterations.rb +1 -1
- data/lib/brute/middleware/020_system_prompt.rb +1 -1
- data/lib/brute/middleware/040_default_compaction_pipeline.rb +356 -0
- data/lib/brute/middleware/{070_tool_pipeline.rb → 070_default_tool_pipeline.rb} +49 -58
- data/lib/brute/prompt_template.rb +19 -15
- data/lib/brute/prompts/base.rb +19 -10
- data/lib/brute/prompts/environment.rb +3 -1
- data/lib/brute/prompts/instructions.rb +9 -7
- data/lib/brute/prompts/skills.rb +5 -3
- data/lib/brute/rack/adapter.rb +40 -23
- data/lib/brute/skill.rb +136 -89
- data/lib/brute/system_prompt.rb +9 -9
- data/lib/brute/token_counter/approximate.rb +54 -0
- data/lib/brute/token_counter/tiktoken.rb +80 -0
- data/lib/brute/token_counter.rb +150 -0
- data/lib/brute/tool.rb +10 -6
- data/lib/brute/tools/adapter.rb +57 -47
- data/lib/brute/tools/fs/snapshot_store.rb +5 -1
- data/lib/brute/tools/fs_patch.rb +16 -8
- data/lib/brute/tools/fs_read.rb +107 -80
- data/lib/brute/tools/fs_remove.rb +6 -2
- data/lib/brute/tools/fs_search.rb +14 -4
- data/lib/brute/tools/fs_undo.rb +6 -2
- data/lib/brute/tools/fs_write.rb +5 -1
- data/lib/brute/tools/net_fetch.rb +6 -2
- data/lib/brute/tools/question.rb +42 -39
- data/lib/brute/tools/shell.rb +20 -5
- data/lib/brute/tools/skill_load.rb +46 -41
- data/lib/brute/tools/sub_agent.rb +2 -2
- data/lib/brute/tools/todo_write.rb +19 -15
- data/lib/brute/truncation.rb +68 -43
- data/lib/brute/turn/agent_pipeline.rb +23 -13
- data/lib/brute/turn/compaction_pipeline.rb +123 -0
- data/lib/brute/turn/pipeline.rb +87 -96
- data/lib/brute/turn/tool_pipeline.rb +4 -3
- data/lib/brute/usage_detection/llmrb.rb +18 -14
- data/lib/brute/usage_detection/open_router.rb +20 -17
- data/lib/brute/usage_detection/ruby_llm.rb +18 -14
- data/lib/brute/usage_detection/usage.rb +10 -1
- data/lib/brute/utils/diff.rb +18 -10
- data/lib/brute/version.rb +1 -1
- data/lib/brute.rb +25 -12
- data/lib/brute_cli/providers/shell.rb +32 -29
- data/lib/brute_cli/providers/shell_response.rb +20 -18
- metadata +48 -5
- data/lib/brute/middleware/040_compaction_check.rb +0 -157
- data/lib/brute/middleware/event_handler.rb +0 -27
|
@@ -0,0 +1,356 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "bundler/setup"
|
|
4
|
+
require "brute"
|
|
5
|
+
|
|
6
|
+
module Brute
|
|
7
|
+
module Middleware
|
|
8
|
+
# Compacts the conversation once it fills too much of the model's window.
|
|
9
|
+
#
|
|
10
|
+
# This layer owns the *when*: before each call it estimates the size of the
|
|
11
|
+
# conversation, and once it reaches `compact_at` of the window it asks a
|
|
12
|
+
# compactor to bring it down to `compact_to`. What is given up, and in what
|
|
13
|
+
# order, is the compactor's business.
|
|
14
|
+
#
|
|
15
|
+
# use Brute::Middleware::Loop::ToolResult
|
|
16
|
+
# use Brute::Middleware::DefaultCompactionPipeline,
|
|
17
|
+
# window: 200_000,
|
|
18
|
+
# summariser: Brute::Completion::OpenRouter.new(config: { access_token: key })
|
|
19
|
+
# use Brute::Middleware::DefaultToolPipeline, tools: tools
|
|
20
|
+
#
|
|
21
|
+
# The compactor it builds is the usual ladder, cheapest first:
|
|
22
|
+
#
|
|
23
|
+
# use Brute::Compaction::Middleware::ToolResults rewrite older tool output
|
|
24
|
+
# use Brute::Compaction::Middleware::SlidingWindow drop the oldest turns, then steps
|
|
25
|
+
# run Brute::Compaction::Summarize summarise what is left, until it fits
|
|
26
|
+
#
|
|
27
|
+
# Pass `compactor:` to say something else. A Brute::Turn::CompactionPipeline
|
|
28
|
+
# is the usual thing to pass, but anything answering
|
|
29
|
+
# `#compact(messages, target:)` will do. Passing no summariser leaves the
|
|
30
|
+
# terminal doing nothing, which is how an agent says it would rather live
|
|
31
|
+
# with a full context than pay to shrink it.
|
|
32
|
+
#
|
|
33
|
+
# It belongs inside the tool loop rather than around it, so it runs before
|
|
34
|
+
# every call rather than once a turn -- a long run of tool results can fill
|
|
35
|
+
# the window without the turn ever ending.
|
|
36
|
+
#
|
|
37
|
+
# Sizing anchors on what the provider itself counted for the last call and
|
|
38
|
+
# measures locally only what has been appended since, so the estimate is
|
|
39
|
+
# exact for the bulk of the conversation and approximate only for its tail.
|
|
40
|
+
#
|
|
41
|
+
# Tool schemas ride in every request and are part of what fills a window,
|
|
42
|
+
# so they are counted too -- against the trigger when nothing has been
|
|
43
|
+
# reported yet, and off the target the compactor is given, because what
|
|
44
|
+
# the schemas occupy is not room the conversation may have. This layer
|
|
45
|
+
# sits above the tool pipeline, so on the very first call of a turn
|
|
46
|
+
# env[:tools] is not set yet: pass `tools:` to have them counted then.
|
|
47
|
+
#
|
|
48
|
+
# Compaction is lossy, and this layer keeps no record of what it gave up:
|
|
49
|
+
# it rewrites env[:messages] and says so with :compact_end. An application
|
|
50
|
+
# that keeps a transcript preserves it from there.
|
|
51
|
+
#
|
|
52
|
+
# agent.on(:compact_end) { |env, payload| archive(env, payload) }
|
|
53
|
+
#
|
|
54
|
+
class DefaultCompactionPipeline < Brute::Middleware::Base
|
|
55
|
+
def initialize(app, window:, summariser: nil, compactor: nil, keep_steps: 2,
|
|
56
|
+
compact_at: 0.7, compact_to: 0.4, token_counter: nil, tools: nil)
|
|
57
|
+
unless compact_to.positive? && compact_to < compact_at && compact_at <= 1
|
|
58
|
+
raise ArgumentError, "expected 0 < compact_to < compact_at <= 1, got #{compact_to} and #{compact_at}"
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
@app = app
|
|
62
|
+
@compactor = compactor || self.class.compactor(summariser: summariser, keep_steps: keep_steps)
|
|
63
|
+
@window = window
|
|
64
|
+
@compact_at = compact_at
|
|
65
|
+
@compact_to = compact_to
|
|
66
|
+
@token_counter = token_counter || Brute::TokenCounter.default
|
|
67
|
+
@tools = tools
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def call(env)
|
|
71
|
+
env[:compactor] = @compactor
|
|
72
|
+
compact(env)
|
|
73
|
+
@app.call(env)
|
|
74
|
+
env
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# The ladder this layer wires when it is not given one.
|
|
78
|
+
def self.compactor(summariser: nil, keep_steps: 2)
|
|
79
|
+
terminal = Declines.new
|
|
80
|
+
|
|
81
|
+
unless summariser.nil?
|
|
82
|
+
terminal = Brute::Compaction::Summarize.new(summariser, keep_steps: keep_steps)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
Brute::Turn::CompactionPipeline.new do
|
|
86
|
+
use Brute::Compaction::Middleware::ToolResults, keep_steps: 1
|
|
87
|
+
use Brute::Compaction::Middleware::SlidingWindow, keep_steps: keep_steps
|
|
88
|
+
|
|
89
|
+
run terminal
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# The floor of a pipeline given nothing to summarise with: what the free
|
|
94
|
+
# layers managed is what the turn gets.
|
|
95
|
+
class Declines
|
|
96
|
+
def call(env) = env
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
private
|
|
100
|
+
|
|
101
|
+
def compact(env)
|
|
102
|
+
context = estimate(env)
|
|
103
|
+
|
|
104
|
+
env.emit_trace do |env|
|
|
105
|
+
if context >= @window * @compact_at
|
|
106
|
+
env.emit(COMPACT_START_EVENT, { context: context })
|
|
107
|
+
|
|
108
|
+
before = @token_counter.count(env[:messages])
|
|
109
|
+
compacted = attempt(env, target(env))
|
|
110
|
+
|
|
111
|
+
unless compacted.nil?
|
|
112
|
+
env[:messages].replace(compacted)
|
|
113
|
+
# The reported total describes a conversation that no longer
|
|
114
|
+
# exists, so the next estimate must not build on it. The call
|
|
115
|
+
# below refreshes it; one that fails leaves the estimate to be
|
|
116
|
+
# counted from scratch instead of anchored to a fiction.
|
|
117
|
+
env[:metadata]&.delete(:last_llm_usage)
|
|
118
|
+
env.emit(
|
|
119
|
+
COMPACT_END_EVENT,
|
|
120
|
+
{ context: context, before: before, after: @token_counter.count(compacted) },
|
|
121
|
+
)
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
# What the conversation alone may occupy: the target, less whatever
|
|
128
|
+
# the schemas are already taking out of it. A window too small to
|
|
129
|
+
# hold its own tool schemas is a configuration a compactor cannot
|
|
130
|
+
# fix, so the floor is 1 rather than a negative target.
|
|
131
|
+
def target(env)
|
|
132
|
+
[(@window * @compact_to).to_i - schemas(env), 1].max
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def schemas(env)
|
|
136
|
+
tools = @tools || env[:tools]
|
|
137
|
+
|
|
138
|
+
if tools.nil?
|
|
139
|
+
0
|
|
140
|
+
else
|
|
141
|
+
@token_counter.count([], tools: tools)
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
# A compactor that raises is a compactor that declined. Compaction is
|
|
146
|
+
# an optimisation, and an agent that cannot shrink its context should
|
|
147
|
+
# carry on with the context it has rather than die of it.
|
|
148
|
+
def attempt(env, target)
|
|
149
|
+
compacted = nil
|
|
150
|
+
|
|
151
|
+
env.emit(COMPACT_DURATION_EVENT, env[:compactor]) do
|
|
152
|
+
compacted = env[:compactor].compact(
|
|
153
|
+
env[:messages],
|
|
154
|
+
target: target,
|
|
155
|
+
token_counter: @token_counter,
|
|
156
|
+
)
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
compacted
|
|
160
|
+
rescue => error
|
|
161
|
+
env.emit(COMPACT_FAILURE_EVENT, error)
|
|
162
|
+
nil
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# What the provider counted for the last call already covers the
|
|
166
|
+
# system prompt, the tool schemas and the chat-template overhead, so
|
|
167
|
+
# only what has landed since it answered needs counting here.
|
|
168
|
+
def estimate(env)
|
|
169
|
+
Brute::TokenCounter.estimate(env, counter: @token_counter, tools: @tools || env[:tools])
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
end
|
|
174
|
+
|
|
175
|
+
__END__
|
|
176
|
+
|
|
177
|
+
describe "brute/middleware/040_default_compaction_pipeline" do
|
|
178
|
+
def conversation
|
|
179
|
+
Brute.log.tap do |log|
|
|
180
|
+
log.user("ask" + ("x" * 1_000))
|
|
181
|
+
log.assistant("answer" + ("y" * 1_000))
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def usage(total) = { last_llm_usage: Brute::UsageDetection::Usage.new(total: total) }
|
|
186
|
+
|
|
187
|
+
def history(turns: 4, size: 6_000)
|
|
188
|
+
Brute.log.tap do |log|
|
|
189
|
+
log.system("instructions")
|
|
190
|
+
turns.times do |n|
|
|
191
|
+
log.user("question #{n}")
|
|
192
|
+
log << Brute::Message.new(role: :assistant, content: n.to_s * size)
|
|
193
|
+
end
|
|
194
|
+
log.user("the current task")
|
|
195
|
+
log << Brute::Message.new(role: :assistant, content: "z" * size)
|
|
196
|
+
end
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def traced(env, hooks = Brute::Hooks::Registry.new)
|
|
200
|
+
Brute::Hooks::Trace.new(env, hooks: hooks)
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def compactor(&block)
|
|
204
|
+
Object.new.tap do |double|
|
|
205
|
+
double.define_singleton_method(:compact) { |messages, target:, **| block.call(messages, target) }
|
|
206
|
+
end
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
it "compacts only once the window fills, and says what the turn gave up" do
|
|
210
|
+
asked = []
|
|
211
|
+
events = []
|
|
212
|
+
shrinks = compactor { |messages, target| asked << target; messages.first(1) }
|
|
213
|
+
declines = compactor { |_messages, target| asked << target; nil }
|
|
214
|
+
raises = compactor { |_messages, _target| raise IOError, "the summariser is down" }
|
|
215
|
+
|
|
216
|
+
hooks = Brute::Hooks::Registry.new
|
|
217
|
+
%i[compact_end compact_failure].each { |event| hooks.on(event) { |_env, *extras| events << [event, *extras.first(1)] } }
|
|
218
|
+
hooks.on(:compact_duration) { |_env, *| events << [:compact_duration] }
|
|
219
|
+
|
|
220
|
+
layer = lambda do |compactor|
|
|
221
|
+
Brute::Middleware::DefaultCompactionPipeline.new(
|
|
222
|
+
->(env) { env },
|
|
223
|
+
compactor: compactor,
|
|
224
|
+
window: 1_000,
|
|
225
|
+
compact_at: 0.7,
|
|
226
|
+
compact_to: 0.4,
|
|
227
|
+
)
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
env = traced({ messages: conversation, metadata: usage(900) }, hooks)
|
|
231
|
+
layer.call(shrinks).call(env)
|
|
232
|
+
|
|
233
|
+
asked.should == [400]
|
|
234
|
+
env[:messages].map(&:role).should == [:user]
|
|
235
|
+
env[:compactor].equal?(shrinks).should.be.true
|
|
236
|
+
events.select { |e, _| e == :compact_end }.should == [[:compact_end, { context: 900, before: 514, after: 256 }]]
|
|
237
|
+
# The attempt is timed, so the compactor that spends a model call is visible.
|
|
238
|
+
events.select { |e, _| e == :compact_duration }.length.should == 1
|
|
239
|
+
|
|
240
|
+
# Sizing anchors on what the provider counted: under compact_at nothing is
|
|
241
|
+
# asked and nothing is said.
|
|
242
|
+
asked.clear
|
|
243
|
+
events.clear
|
|
244
|
+
layer.call(shrinks).call(traced({ messages: conversation, metadata: usage(600) }, hooks))
|
|
245
|
+
asked.should == []
|
|
246
|
+
events.should == []
|
|
247
|
+
|
|
248
|
+
# ...but what landed since it answered is counted on top of it, and that
|
|
249
|
+
# is what tips this one over.
|
|
250
|
+
tail = conversation.tap { |log| log.tool("z" * 400, tool_call_id: "tc1") }
|
|
251
|
+
layer.call(declines).call(traced({ messages: tail, metadata: usage(600) }, hooks))
|
|
252
|
+
asked.should == [400]
|
|
253
|
+
|
|
254
|
+
# With nothing reported at all, the whole conversation is counted here.
|
|
255
|
+
asked.clear
|
|
256
|
+
layer.call(declines).call(traced({ messages: conversation }, hooks))
|
|
257
|
+
asked.should == []
|
|
258
|
+
|
|
259
|
+
# A compactor that raises is a compactor that declined: reported, not
|
|
260
|
+
# fatal. An agent that cannot shrink its context carries on with it.
|
|
261
|
+
events.clear
|
|
262
|
+
failing = traced({ messages: conversation, metadata: usage(900) }, hooks)
|
|
263
|
+
should.not.raise(IOError) { layer.call(raises).call(failing) }
|
|
264
|
+
failing[:messages].length.should == 2
|
|
265
|
+
events.select { |e, _| e == :compact_failure }.map { |_, error| error.message }
|
|
266
|
+
.should == ["the summariser is down"]
|
|
267
|
+
|
|
268
|
+
# A target at or above the trigger would compact on every single step.
|
|
269
|
+
should.raise(ArgumentError) do
|
|
270
|
+
Brute::Middleware::DefaultCompactionPipeline.new(->(e) { e }, compactor: shrinks, window: 10,
|
|
271
|
+
compact_at: 0.4, compact_to: 0.7)
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
it "wires the usual ladder, cheapest first, and only pays when the free layers cannot" do
|
|
276
|
+
rounds = 0
|
|
277
|
+
summariser = lambda do |env|
|
|
278
|
+
rounds += 1
|
|
279
|
+
env[:messages] << Brute::Message.new(role: :assistant, content: "what happened")
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
ladder = Brute::Middleware::DefaultCompactionPipeline.compactor(summariser: summariser, keep_steps: 1)
|
|
283
|
+
ladder.should.be.kind_of Brute::Turn::CompactionPipeline
|
|
284
|
+
|
|
285
|
+
messages = history
|
|
286
|
+
before = Brute::Compaction::Transcript.tokens(messages)
|
|
287
|
+
out = ladder.compact(messages, target: 1_500)
|
|
288
|
+
|
|
289
|
+
Brute::Compaction::Transcript.tokens(out).should.be < before
|
|
290
|
+
|
|
291
|
+
# Given no summariser the terminal declines, and the free layers are all
|
|
292
|
+
# the agent gets -- a policy said out loud rather than configured.
|
|
293
|
+
free = Brute::Middleware::DefaultCompactionPipeline.compactor(summariser: nil, keep_steps: 1)
|
|
294
|
+
free.compact(history, target: 1_500).should.not.be.nil
|
|
295
|
+
|
|
296
|
+
# One layer, so it still owns the *when* as well as the what.
|
|
297
|
+
layer = Brute::Middleware::DefaultCompactionPipeline.new(->(e) { e }, window: 1_000, summariser: summariser)
|
|
298
|
+
env = traced({ messages: history, metadata: {} })
|
|
299
|
+
layer.call(env)
|
|
300
|
+
env[:messages].length.should.be < 11
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
it "counts the schemas it will send, and stops anchoring on a conversation that is gone" do
|
|
304
|
+
asked = []
|
|
305
|
+
shrinks = compactor { |messages, target| asked << target; messages.first(1) }
|
|
306
|
+
|
|
307
|
+
tool = Brute::Turn::ToolPipeline.new(name: "search", description: "search the web. " * 40) do
|
|
308
|
+
run ->(env) { env[:result] = "" }
|
|
309
|
+
end
|
|
310
|
+
schemas = Brute::TokenCounter.default.count([], tools: [tool])
|
|
311
|
+
|
|
312
|
+
layer = lambda do |**options|
|
|
313
|
+
Brute::Middleware::DefaultCompactionPipeline.new(
|
|
314
|
+
->(env) { env },
|
|
315
|
+
compactor: shrinks,
|
|
316
|
+
window: 1_000,
|
|
317
|
+
**options,
|
|
318
|
+
).tap { |it| it.define_singleton_method(:emit) { |_event, _env, *, &work| work&.call } }
|
|
319
|
+
end
|
|
320
|
+
|
|
321
|
+
# 650 tokens of conversation, nothing reported yet: under the 700 the
|
|
322
|
+
# window triggers at, so on its own it is left alone.
|
|
323
|
+
said = -> { Brute.log.tap { |log| log.user("x" * 2_578) } }
|
|
324
|
+
Brute::TokenCounter.default.count(said.call).should == 650
|
|
325
|
+
|
|
326
|
+
layer.call.call(traced({ messages: said.call, metadata: {} }))
|
|
327
|
+
asked.should == []
|
|
328
|
+
|
|
329
|
+
# The same conversation ships with tool schemas in every request, and
|
|
330
|
+
# counting what they occupy is what tips it over. What is left of the
|
|
331
|
+
# target after them is what the compactor is asked for -- the schemas are
|
|
332
|
+
# not room the conversation may have.
|
|
333
|
+
layer.call(tools: [tool]).call(traced({ messages: said.call, metadata: {} }))
|
|
334
|
+
asked.should == [400 - schemas]
|
|
335
|
+
|
|
336
|
+
# Reported usage already covers the schemas, so the warm path must not
|
|
337
|
+
# count them again -- 600 reported and nothing said since is 600, tools
|
|
338
|
+
# or no tools.
|
|
339
|
+
asked.clear
|
|
340
|
+
layer.call(tools: [tool]).call(traced({ messages: conversation, metadata: usage(600) }))
|
|
341
|
+
asked.should == []
|
|
342
|
+
|
|
343
|
+
# What the provider counted describes the conversation as it was. Once
|
|
344
|
+
# compaction has given part of it up that number is a fiction, so it goes
|
|
345
|
+
# -- the next estimate counts from scratch rather than building on it.
|
|
346
|
+
compacted = traced({ messages: conversation, metadata: usage(900) })
|
|
347
|
+
layer.call.call(compacted)
|
|
348
|
+
compacted[:messages].length.should == 1
|
|
349
|
+
compacted[:metadata].key?(:last_llm_usage).should.be.false
|
|
350
|
+
|
|
351
|
+
# Nothing given up, nothing invalidated.
|
|
352
|
+
kept = traced({ messages: conversation, metadata: usage(900) })
|
|
353
|
+
layer.call(compactor: compactor { |_messages, _target| nil }).call(kept)
|
|
354
|
+
kept[:metadata].key?(:last_llm_usage).should.be.true
|
|
355
|
+
end
|
|
356
|
+
end
|
|
@@ -8,7 +8,7 @@ require "async/barrier"
|
|
|
8
8
|
|
|
9
9
|
module Brute
|
|
10
10
|
module Middleware
|
|
11
|
-
class
|
|
11
|
+
class DefaultToolPipeline < Brute::Middleware::Base
|
|
12
12
|
def initialize(app, tools: [])
|
|
13
13
|
@app = app
|
|
14
14
|
@tools = tools
|
|
@@ -33,7 +33,7 @@ module Brute
|
|
|
33
33
|
tools_to_run = Array(tools_to_run).reject { |tc| tc.name == "question" }
|
|
34
34
|
|
|
35
35
|
available_tools = Brute::Tools::Adapter.wrap_all(env[:tools])
|
|
36
|
-
env
|
|
36
|
+
env.emit(TOOL_CALLS_EVENT, tools_to_run)
|
|
37
37
|
|
|
38
38
|
results = []
|
|
39
39
|
|
|
@@ -44,47 +44,58 @@ module Brute
|
|
|
44
44
|
barrier = Async::Barrier.new
|
|
45
45
|
|
|
46
46
|
tools_to_run.each do |tool_call|
|
|
47
|
-
barrier.async do
|
|
47
|
+
barrier.async do
|
|
48
48
|
name = tool_call.name.to_sym
|
|
49
49
|
args = tool_call.arguments
|
|
50
50
|
|
|
51
|
-
# Lifecycle hooks (Brute::Hooks):
|
|
52
|
-
# :arguments or short-circuit with a :result;
|
|
53
|
-
# denies on a false (or String) return;
|
|
51
|
+
# Lifecycle hooks (Brute::Hooks): tool_start may rewrite
|
|
52
|
+
# :arguments or short-circuit with a :result; tool_approve
|
|
53
|
+
# denies on a false (or String) return; tool_end may
|
|
54
54
|
# rewrite :result.
|
|
55
55
|
call_env = {
|
|
56
56
|
name: name.to_s,
|
|
57
57
|
arguments: args,
|
|
58
58
|
result: nil,
|
|
59
59
|
denied: nil,
|
|
60
|
-
events: env[:events],
|
|
61
60
|
metadata: {},
|
|
62
61
|
turn_env: env,
|
|
63
62
|
}
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
63
|
+
env.emit_trace do |env|
|
|
64
|
+
# A subscriber takes part by mutating the call env: set
|
|
65
|
+
# :result to answer without executing, set :denied to refuse.
|
|
66
|
+
env.emit(TOOL_START_EVENT, call_env)
|
|
67
|
+
|
|
68
|
+
if call_env[:result].nil?
|
|
69
|
+
env.emit(TOOL_APPROVE_EVENT, call_env)
|
|
70
|
+
|
|
71
|
+
if (denial = call_env[:denied])
|
|
72
|
+
if denial.is_a?(String)
|
|
73
|
+
call_env[:result] = denial
|
|
74
|
+
else
|
|
75
|
+
call_env[:result] = %(Tool call to "#{name}" was denied.)
|
|
76
|
+
end
|
|
77
|
+
end
|
|
73
78
|
end
|
|
74
|
-
end
|
|
75
79
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
80
|
+
# Only the tool's own execution is timed: a call that
|
|
81
|
+
# tool_start answered, or tool_approve denied, never ran.
|
|
82
|
+
result = call_env[:result]
|
|
79
83
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
84
|
+
if result.nil?
|
|
85
|
+
env.emit(TOOL_DURATION_EVENT, call_env) do
|
|
86
|
+
result = available_tools[name].call(call_env[:arguments])
|
|
87
|
+
end
|
|
83
88
|
end
|
|
84
|
-
end
|
|
85
89
|
|
|
86
|
-
|
|
87
|
-
|
|
90
|
+
call_env[:result] = result
|
|
91
|
+
env.emit(TOOL_END_EVENT, call_env)
|
|
92
|
+
rescue => error
|
|
93
|
+
# Capture the error as a tool result so the LLM can see it
|
|
94
|
+
# and reason about the failure, rather than crashing the
|
|
95
|
+
# entire middleware chain.
|
|
96
|
+
env.emit(TOOL_FAILURE_EVENT, call_env, error)
|
|
97
|
+
call_env[:result] = "Error: #{error.class}: #{error.message}"
|
|
98
|
+
end
|
|
88
99
|
result = call_env[:result]
|
|
89
100
|
|
|
90
101
|
# Coerce to String so Hash results (e.g. Shell's
|
|
@@ -101,12 +112,6 @@ module Brute
|
|
|
101
112
|
end
|
|
102
113
|
|
|
103
114
|
results << [tool_call, content]
|
|
104
|
-
rescue => e
|
|
105
|
-
# Capture the error as a tool result so the LLM can see it
|
|
106
|
-
# and reason about the failure, rather than crashing the
|
|
107
|
-
# entire middleware chain.
|
|
108
|
-
env[:events] << { type: :error, data: { error: e, message: e.message } }
|
|
109
|
-
results << [tool_call, "Error: #{e.class}: #{e.message}"]
|
|
110
115
|
end
|
|
111
116
|
end
|
|
112
117
|
|
|
@@ -120,7 +125,6 @@ module Brute
|
|
|
120
125
|
results.sort_by! { |tool_call, _| tools_to_run.index(tool_call) }
|
|
121
126
|
|
|
122
127
|
results.each do |tool_call, content|
|
|
123
|
-
env[:events] << { type: :tool_result, data: { name: tool_call.name, content: content } }
|
|
124
128
|
env[:messages] << Brute::Message.new(role: :tool, content: content, tool_call_id: tool_call.id)
|
|
125
129
|
end
|
|
126
130
|
end
|
|
@@ -129,26 +133,13 @@ module Brute
|
|
|
129
133
|
end
|
|
130
134
|
|
|
131
135
|
private
|
|
132
|
-
|
|
133
|
-
def on_tool_call_start_event(pending_tools)
|
|
134
|
-
{
|
|
135
|
-
type: :tool_call_start,
|
|
136
|
-
data: pending_tools.map { |tc|
|
|
137
|
-
{
|
|
138
|
-
name: tc.name,
|
|
139
|
-
call_id: tc.id,
|
|
140
|
-
arguments: tc.arguments
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
end
|
|
145
136
|
end
|
|
146
137
|
end
|
|
147
138
|
end
|
|
148
139
|
|
|
149
140
|
__END__
|
|
150
141
|
|
|
151
|
-
describe "brute/middleware/
|
|
142
|
+
describe "brute/middleware/070_default_tool_pipeline" do
|
|
152
143
|
require "brute/messages"
|
|
153
144
|
require "brute/truncation"
|
|
154
145
|
|
|
@@ -156,7 +147,7 @@ describe "brute/middleware/070_tool_pipeline" do
|
|
|
156
147
|
inner = ->(env) {
|
|
157
148
|
env[:messages] << Brute::Message.new(role: :assistant, content: "hi")
|
|
158
149
|
}
|
|
159
|
-
mw = Brute::Middleware::
|
|
150
|
+
mw = Brute::Middleware::DefaultToolPipeline.new(inner, tools: [])
|
|
160
151
|
env = {
|
|
161
152
|
messages: Brute.log,
|
|
162
153
|
events: [],
|
|
@@ -170,7 +161,7 @@ describe "brute/middleware/070_tool_pipeline" do
|
|
|
170
161
|
seen = nil
|
|
171
162
|
inner = ->(env) { seen = env[:tools] }
|
|
172
163
|
tool = { name: "echo", description: "", execute: ->(**) { "ok" } }
|
|
173
|
-
mw = Brute::Middleware::
|
|
164
|
+
mw = Brute::Middleware::DefaultToolPipeline.new(inner, tools: [tool])
|
|
174
165
|
env = { messages: Brute.log, events: [] }
|
|
175
166
|
env[:messages].user("hi")
|
|
176
167
|
mw.call(env)
|
|
@@ -183,7 +174,7 @@ describe "brute/middleware/070_tool_pipeline" do
|
|
|
183
174
|
# builds a real pipeline rather than instantiating the middleware alone.
|
|
184
175
|
def hooked(inner, tools:, &subscribe)
|
|
185
176
|
pipeline = Brute::Turn::Pipeline.new
|
|
186
|
-
pipeline.use Brute::Middleware::
|
|
177
|
+
pipeline.use Brute::Middleware::DefaultToolPipeline, tools: tools
|
|
187
178
|
pipeline.run(Object.new.tap { |o| o.define_singleton_method(:call, &inner) })
|
|
188
179
|
subscribe.call(pipeline)
|
|
189
180
|
pipeline
|
|
@@ -193,7 +184,7 @@ describe "brute/middleware/070_tool_pipeline" do
|
|
|
193
184
|
{ messages: Brute.log, events: [] }
|
|
194
185
|
end
|
|
195
186
|
|
|
196
|
-
it "
|
|
187
|
+
it "tool_start may rewrite arguments and short-circuit with a result" do
|
|
197
188
|
tool = { name: "echo", description: "", execute: ->(text:) { "ran:#{text}" } }
|
|
198
189
|
inner = ->(env) do
|
|
199
190
|
env[:messages] << Brute::Message.new(role: :assistant, content: "",
|
|
@@ -201,41 +192,41 @@ describe "brute/middleware/070_tool_pipeline" do
|
|
|
201
192
|
end
|
|
202
193
|
|
|
203
194
|
pipeline = hooked(inner, tools: [tool]) do |p|
|
|
204
|
-
p.on(:
|
|
195
|
+
p.on(:tool_start) { |_env, call| call[:arguments] = { text: "rewritten" } }
|
|
205
196
|
end
|
|
206
197
|
env = hook_env
|
|
207
198
|
env[:messages].user("hi")
|
|
208
199
|
pipeline.call(env)
|
|
209
200
|
env[:messages].last.content.should == "ran:rewritten"
|
|
210
201
|
|
|
211
|
-
canned = hooked(inner, tools: [tool]) { |p| p.on(:
|
|
202
|
+
canned = hooked(inner, tools: [tool]) { |p| p.on(:tool_start) { |_env, call| call[:result] = "canned" } }
|
|
212
203
|
env2 = hook_env
|
|
213
204
|
env2[:messages].user("hi")
|
|
214
205
|
canned.call(env2)
|
|
215
206
|
env2[:messages].last.content.should == "canned" # never executed
|
|
216
207
|
end
|
|
217
208
|
|
|
218
|
-
it "
|
|
209
|
+
it "tool_approve denies on false (generic message) or String (custom)" do
|
|
219
210
|
tool = { name: "exec", description: "", execute: ->(**) { "ran" } }
|
|
220
211
|
inner = ->(env) do
|
|
221
212
|
env[:messages] << Brute::Message.new(role: :assistant, content: "",
|
|
222
213
|
tool_calls: [{ id: "tc1", name: "exec", arguments: {} }])
|
|
223
214
|
end
|
|
224
215
|
|
|
225
|
-
denied = hooked(inner, tools: [tool]) { |p| p.on(:
|
|
216
|
+
denied = hooked(inner, tools: [tool]) { |p| p.on(:tool_approve) { |_env, call| call[:denied] = true } }
|
|
226
217
|
env = hook_env
|
|
227
218
|
env[:messages].user("hi")
|
|
228
219
|
denied.call(env)
|
|
229
220
|
env[:messages].last.content.should == %(Tool call to "exec" was denied.)
|
|
230
221
|
|
|
231
|
-
by_policy = hooked(inner, tools: [tool]) { |p| p.on(:
|
|
222
|
+
by_policy = hooked(inner, tools: [tool]) { |p| p.on(:tool_approve) { |_env, call| call[:denied] = "denied by policy" } }
|
|
232
223
|
env2 = hook_env
|
|
233
224
|
env2[:messages].user("hi")
|
|
234
225
|
by_policy.call(env2)
|
|
235
226
|
env2[:messages].last.content.should == "denied by policy"
|
|
236
227
|
end
|
|
237
228
|
|
|
238
|
-
it "
|
|
229
|
+
it "tool_end may rewrite the result" do
|
|
239
230
|
tool = { name: "echo", description: "", execute: ->(**) { "raw" } }
|
|
240
231
|
inner = ->(env) do
|
|
241
232
|
env[:messages] << Brute::Message.new(role: :assistant, content: "",
|
|
@@ -243,7 +234,7 @@ describe "brute/middleware/070_tool_pipeline" do
|
|
|
243
234
|
end
|
|
244
235
|
|
|
245
236
|
pipeline = hooked(inner, tools: [tool]) do |p|
|
|
246
|
-
p.on(:
|
|
237
|
+
p.on(:tool_end) { |_env, call| call[:result] = "rewrote(#{call[:result]})" }
|
|
247
238
|
end
|
|
248
239
|
env = hook_env
|
|
249
240
|
env[:messages].user("hi")
|
|
@@ -56,25 +56,29 @@ module Brute
|
|
|
56
56
|
|
|
57
57
|
private
|
|
58
58
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
59
|
+
def locals(ctx)
|
|
60
|
+
@section_keys.to_h do |key|
|
|
61
|
+
value = self[key]
|
|
62
|
+
if value.is_a?(Proc)
|
|
63
|
+
resolved = (value.arity.zero? ? value.call : value.call(ctx))
|
|
64
|
+
else
|
|
65
|
+
resolved = value
|
|
66
|
+
end
|
|
67
|
+
[key, resolved]
|
|
68
|
+
end.merge(ctx: ctx)
|
|
69
|
+
end
|
|
66
70
|
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
71
|
+
def render(values)
|
|
72
|
+
context = binding
|
|
73
|
+
values.each { |key, value| context.local_variable_set(key, value) }
|
|
74
|
+
ERB.new(template_source, trim_mode: "-").result(context)
|
|
75
|
+
end
|
|
72
76
|
|
|
73
77
|
# A path that exists is re-read every time; anything else is treated as
|
|
74
78
|
# an inline ERB source string.
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
79
|
+
def template_source
|
|
80
|
+
File.exist?(@template.to_s) ? File.read(@template) : @template.to_s
|
|
81
|
+
end
|
|
78
82
|
end
|
|
79
83
|
end
|
|
80
84
|
|