brute 5.1.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/brute/compaction/middleware/sliding_window.rb +1 -1
- data/lib/brute/compaction/middleware/strategy.rb +0 -2
- data/lib/brute/compaction/summarize.rb +30 -21
- data/lib/brute/completion/lang_chain.rb +34 -31
- data/lib/brute/completion/llmrb.rb +31 -29
- data/lib/brute/completion/open_router.rb +43 -40
- data/lib/brute/completion/ruby_llm.rb +28 -27
- data/lib/brute/contrib/otel.rb +81 -49
- data/lib/brute/eval/case.rb +7 -2
- data/lib/brute/eval/suite.rb +10 -5
- data/lib/brute/eval/transcript.rb +3 -3
- data/lib/brute/eval/world.rb +2 -2
- data/lib/brute/eval.rb +1 -1
- data/lib/brute/events/handler.rb +3 -1
- data/lib/brute/events/prefixed_terminal_output.rb +3 -1
- data/lib/brute/events/terminal_output_handler.rb +1 -1
- data/lib/brute/hooks.rb +88 -112
- data/lib/brute/message_transport/anthropic.rb +13 -3
- data/lib/brute/message_transport/llm.rb +8 -4
- data/lib/brute/message_transport/open_router.rb +30 -16
- data/lib/brute/message_transport/openai.rb +14 -7
- data/lib/brute/message_transport/ruby_llm.rb +21 -17
- data/lib/brute/message_transport/ruby_open_ai.rb +59 -57
- data/lib/brute/messages.rb +15 -8
- data/lib/brute/middleware/000_base.rb +5 -5
- data/lib/brute/middleware/002_session_log.rb +12 -4
- data/lib/brute/middleware/008_checkpoint.rb +26 -18
- data/lib/brute/middleware/010_max_iterations.rb +1 -1
- data/lib/brute/middleware/020_system_prompt.rb +1 -1
- data/lib/brute/middleware/040_default_compaction_pipeline.rb +49 -45
- data/lib/brute/middleware/070_default_tool_pipeline.rb +44 -53
- data/lib/brute/prompt_template.rb +19 -15
- data/lib/brute/prompts/base.rb +19 -10
- data/lib/brute/prompts/environment.rb +3 -1
- data/lib/brute/prompts/instructions.rb +9 -7
- data/lib/brute/prompts/skills.rb +5 -3
- data/lib/brute/rack/adapter.rb +40 -23
- data/lib/brute/skill.rb +136 -89
- data/lib/brute/system_prompt.rb +9 -9
- data/lib/brute/token_counter.rb +3 -1
- data/lib/brute/tool.rb +10 -6
- data/lib/brute/tools/adapter.rb +53 -43
- data/lib/brute/tools/fs/snapshot_store.rb +5 -1
- data/lib/brute/tools/fs_patch.rb +16 -8
- data/lib/brute/tools/fs_read.rb +107 -80
- data/lib/brute/tools/fs_remove.rb +6 -2
- data/lib/brute/tools/fs_search.rb +14 -4
- data/lib/brute/tools/fs_undo.rb +6 -2
- data/lib/brute/tools/fs_write.rb +5 -1
- data/lib/brute/tools/net_fetch.rb +6 -2
- data/lib/brute/tools/question.rb +42 -39
- data/lib/brute/tools/shell.rb +20 -5
- data/lib/brute/tools/skill_load.rb +44 -39
- data/lib/brute/tools/todo_write.rb +19 -15
- data/lib/brute/truncation.rb +68 -43
- data/lib/brute/turn/agent_pipeline.rb +23 -13
- data/lib/brute/turn/compaction_pipeline.rb +4 -3
- data/lib/brute/turn/pipeline.rb +87 -96
- data/lib/brute/turn/tool_pipeline.rb +4 -3
- data/lib/brute/usage_detection/llmrb.rb +18 -14
- data/lib/brute/usage_detection/open_router.rb +20 -17
- data/lib/brute/usage_detection/ruby_llm.rb +18 -14
- data/lib/brute/usage_detection/usage.rb +10 -1
- data/lib/brute/utils/diff.rb +18 -10
- data/lib/brute/version.rb +1 -1
- data/lib/brute.rb +10 -11
- data/lib/brute_cli/providers/shell.rb +32 -29
- data/lib/brute_cli/providers/shell_response.rb +20 -18
- metadata +30 -5
- data/lib/brute/middleware/040_compaction_check.rb +0 -73
- data/lib/brute/middleware/070_tool_pipeline.rb +0 -61
- data/lib/brute/middleware/event_handler.rb +0 -27
data/lib/brute/contrib/otel.rb
CHANGED
|
@@ -19,8 +19,8 @@ module Brute
|
|
|
19
19
|
# Every event it needs already exists:
|
|
20
20
|
#
|
|
21
21
|
# turn_start / turn_end the span itself
|
|
22
|
-
#
|
|
23
|
-
#
|
|
22
|
+
# llm_end token usage, from env[:metadata][:last_llm_usage]
|
|
23
|
+
# tool_start / tool_end a span event per tool call and result
|
|
24
24
|
#
|
|
25
25
|
# The tracer is injectable; without one it asks OpenTelemetry, and when
|
|
26
26
|
# the SDK is not loaded `subscribe` does nothing at all.
|
|
@@ -35,33 +35,50 @@ module Brute
|
|
|
35
35
|
|
|
36
36
|
class << self
|
|
37
37
|
def subscribe(agent, tracer: default_tracer)
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
38
|
+
if tracer.nil?
|
|
39
|
+
agent
|
|
40
|
+
else
|
|
41
|
+
agent
|
|
42
|
+
.on(Brute::Hooks::TURN_START_EVENT) { |env| start(env, tracer) }
|
|
43
|
+
.on(Brute::Hooks::TURN_END_EVENT) { |env| finish(env) }
|
|
44
|
+
.on(Brute::Hooks::LLM_END_EVENT) { |env| record_usage(env) }
|
|
45
|
+
.on(Brute::Hooks::TOOL_START_EVENT) { |env, call| tool_called(env, call) }
|
|
46
|
+
.on(Brute::Hooks::TOOL_END_EVENT) { |env, call| tool_returned(env, call) }
|
|
47
|
+
.on(Brute::Hooks::LLM_FAILURE_EVENT) { |env| failed(env) }
|
|
48
|
+
.on(Brute::Hooks::MIDDLEWARE_DURATION_EVENT) { |env, started, finished, layer| layer_finished(
|
|
49
|
+
env,
|
|
50
|
+
started,
|
|
51
|
+
finished,
|
|
52
|
+
layer,
|
|
53
|
+
) }
|
|
54
|
+
.on(Brute::Hooks::TURN_DURATION_EVENT) { |env, started, finished| timed(env, "turn", finished - started) }
|
|
55
|
+
.on(Brute::Hooks::LLM_DURATION_EVENT) { |env, started, finished| timed(env, "llm", finished - started) }
|
|
56
|
+
.on(Brute::Hooks::TOOL_DURATION_EVENT) { |env, started, finished, call| timed(
|
|
57
|
+
env,
|
|
58
|
+
"tool",
|
|
59
|
+
finished - started,
|
|
60
|
+
"tool.name" => call[:name].to_s,
|
|
61
|
+
) }
|
|
62
|
+
end
|
|
51
63
|
end
|
|
52
64
|
|
|
53
65
|
def default_tracer
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
66
|
+
if defined?(::OpenTelemetry)
|
|
67
|
+
::OpenTelemetry.tracer_provider.tracer("brute", Brute::VERSION)
|
|
68
|
+
else
|
|
69
|
+
nil
|
|
70
|
+
end
|
|
57
71
|
end
|
|
58
72
|
|
|
59
73
|
private
|
|
60
74
|
|
|
61
75
|
def start(env, tracer)
|
|
62
|
-
env[:span] = tracer.start_span(
|
|
63
|
-
|
|
64
|
-
|
|
76
|
+
env[:span] = tracer.start_span(
|
|
77
|
+
SPAN_NAME,
|
|
78
|
+
attributes: {
|
|
79
|
+
"brute.messages" => env[:messages]&.size,
|
|
80
|
+
}.compact,
|
|
81
|
+
)
|
|
65
82
|
end
|
|
66
83
|
|
|
67
84
|
def finish(env)
|
|
@@ -73,40 +90,55 @@ module Brute
|
|
|
73
90
|
# arrives under the same names, and what it did not report is
|
|
74
91
|
# absent rather than zero.
|
|
75
92
|
def record_usage(env)
|
|
76
|
-
span
|
|
77
|
-
usage = env.dig(:metadata, :last_llm_usage)
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
93
|
+
span = env[:span]
|
|
94
|
+
usage = env.dig(:metadata, :last_llm_usage)
|
|
95
|
+
|
|
96
|
+
if span && usage
|
|
97
|
+
{
|
|
98
|
+
"gen_ai.usage.input_tokens" => usage.input,
|
|
99
|
+
"gen_ai.usage.output_tokens" => usage.output,
|
|
100
|
+
"gen_ai.usage.total_tokens" => usage.total,
|
|
101
|
+
"gen_ai.usage.reasoning_tokens" => usage.reasoning,
|
|
102
|
+
"gen_ai.usage.cache_read_tokens" => usage.cache_read,
|
|
103
|
+
"gen_ai.usage.cost" => usage.cost,
|
|
104
|
+
}.each do |name, value|
|
|
105
|
+
unless value.nil?
|
|
106
|
+
span.set_attribute(name, value)
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
87
110
|
end
|
|
88
111
|
|
|
89
112
|
def tool_called(env, call)
|
|
90
|
-
env[:span]&.add_event(
|
|
91
|
-
"
|
|
92
|
-
|
|
93
|
-
|
|
113
|
+
env[:span]&.add_event(
|
|
114
|
+
"tool_call",
|
|
115
|
+
attributes: {
|
|
116
|
+
"tool.name" => call[:name].to_s,
|
|
117
|
+
"tool.arguments" => call[:arguments].to_s,
|
|
118
|
+
},
|
|
119
|
+
)
|
|
94
120
|
end
|
|
95
121
|
|
|
96
122
|
def tool_returned(env, call)
|
|
97
|
-
env[:span]&.add_event(
|
|
98
|
-
"
|
|
99
|
-
|
|
100
|
-
|
|
123
|
+
env[:span]&.add_event(
|
|
124
|
+
"tool_result",
|
|
125
|
+
attributes: {
|
|
126
|
+
"tool.name" => call[:name].to_s,
|
|
127
|
+
"tool.bytes" => call[:result].to_s.bytesize,
|
|
128
|
+
},
|
|
129
|
+
)
|
|
101
130
|
end
|
|
102
131
|
|
|
103
|
-
# The layer's work is :
|
|
104
|
-
# with it rather than having to be correlated with :
|
|
132
|
+
# The layer's work is :middleware_duration's block, so start and finish arrive
|
|
133
|
+
# with it rather than having to be correlated with :middleware_start by hand.
|
|
105
134
|
def layer_finished(env, started, finished, layer)
|
|
106
|
-
env[:span]&.add_event(
|
|
107
|
-
"middleware
|
|
108
|
-
|
|
109
|
-
|
|
135
|
+
env[:span]&.add_event(
|
|
136
|
+
"middleware",
|
|
137
|
+
attributes: {
|
|
138
|
+
"middleware.name" => layer.class.name.to_s,
|
|
139
|
+
"middleware.duration" => finished - started,
|
|
140
|
+
},
|
|
141
|
+
)
|
|
110
142
|
end
|
|
111
143
|
|
|
112
144
|
# Every pair in the turn has a timed middle, so each one's duration
|
|
@@ -170,8 +202,8 @@ describe "brute/contrib/otel" do
|
|
|
170
202
|
tool_calls: [{ id: "tc1", name: "echo", arguments: { "text" => "hi" } }])
|
|
171
203
|
end
|
|
172
204
|
|
|
173
|
-
emit(Brute::Hooks::LLM_DURATION_EVENT
|
|
174
|
-
emit(Brute::Hooks::
|
|
205
|
+
env.emit(Brute::Hooks::LLM_DURATION_EVENT) { :provider_call }
|
|
206
|
+
env.emit(Brute::Hooks::LLM_END_EVENT)
|
|
175
207
|
env
|
|
176
208
|
end
|
|
177
209
|
end)
|
|
@@ -190,7 +222,7 @@ describe "brute/contrib/otel" do
|
|
|
190
222
|
span.events.map(&:first).should.include "tool_result"
|
|
191
223
|
span.events.find { |name, _| name == "tool_call" }.last["tool.name"].should == "echo"
|
|
192
224
|
|
|
193
|
-
# :
|
|
225
|
+
# :middleware_end is timed, so each layer reports its own duration.
|
|
194
226
|
layer_event = span.events.find { |name, _| name == "middleware" }
|
|
195
227
|
layer_event.last["middleware.name"].should == "Brute::Middleware::DefaultToolPipeline"
|
|
196
228
|
layer_event.last["middleware.duration"].should.be >= 0
|
data/lib/brute/eval/case.rb
CHANGED
|
@@ -8,7 +8,12 @@ module Brute
|
|
|
8
8
|
# The budget a case allows itself. Lenient on purpose: an agent that took
|
|
9
9
|
# one search too many is worth knowing about, but it is not the same
|
|
10
10
|
# fault as an agent that answered wrongly.
|
|
11
|
-
Budget = Data.define(
|
|
11
|
+
Budget = Data.define(
|
|
12
|
+
:iterations,
|
|
13
|
+
:tool_calls,
|
|
14
|
+
:tokens,
|
|
15
|
+
:seconds,
|
|
16
|
+
) do
|
|
12
17
|
def initialize(iterations: 10, tool_calls: 8, tokens: 100_000, seconds: 180)
|
|
13
18
|
super
|
|
14
19
|
end
|
|
@@ -94,7 +99,7 @@ module Brute
|
|
|
94
99
|
said_it(transcript),
|
|
95
100
|
afforded(transcript),
|
|
96
101
|
checked(transcript),
|
|
97
|
-
].flatten.compact.uniq
|
|
102
|
+
].flatten.compact.uniq,
|
|
98
103
|
)
|
|
99
104
|
end
|
|
100
105
|
|
data/lib/brute/eval/suite.rb
CHANGED
|
@@ -21,7 +21,12 @@ module Brute
|
|
|
21
21
|
#
|
|
22
22
|
# #run answers a process exit status, so an eval script ends `exit(...)`.
|
|
23
23
|
class Suite
|
|
24
|
-
Result = Data.define(
|
|
24
|
+
Result = Data.define(
|
|
25
|
+
:kase,
|
|
26
|
+
:run,
|
|
27
|
+
:transcript,
|
|
28
|
+
:verdict,
|
|
29
|
+
)
|
|
25
30
|
|
|
26
31
|
def initialize(agent:, cases:, world: World.new, out: $stdout)
|
|
27
32
|
@agent = agent
|
|
@@ -67,10 +72,10 @@ module Brute
|
|
|
67
72
|
transcript.published = @world.published.dup
|
|
68
73
|
|
|
69
74
|
Result.new(
|
|
70
|
-
kase:
|
|
71
|
-
run:
|
|
75
|
+
kase: kase,
|
|
76
|
+
run: run,
|
|
72
77
|
transcript: transcript,
|
|
73
|
-
verdict:
|
|
78
|
+
verdict: kase.verdict(transcript),
|
|
74
79
|
)
|
|
75
80
|
end
|
|
76
81
|
|
|
@@ -93,7 +98,7 @@ module Brute
|
|
|
93
98
|
" spent: #{transcript.iterations}/#{budget.iterations} iterations, " \
|
|
94
99
|
"#{transcript.calls.length}/#{budget.tool_calls} calls, " \
|
|
95
100
|
"#{transcript.tokens}/#{budget.tokens} tokens, " \
|
|
96
|
-
"#{transcript.seconds.round(1)}s"
|
|
101
|
+
"#{transcript.seconds.round(1)}s",
|
|
97
102
|
)
|
|
98
103
|
result.verdict.failures.each { |failure| @out.puts " - #{failure}" }
|
|
99
104
|
@out.puts " said:"
|
|
@@ -34,12 +34,12 @@ module Brute
|
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
# The call env the tool pipeline hands its subscribers is one mutable
|
|
37
|
-
# hash per call, so the entry kept here at :
|
|
37
|
+
# hash per call, so the entry kept here at :tool_start carries the
|
|
38
38
|
# result the tool answered with by the time anyone reads it.
|
|
39
39
|
def subscribe(agent)
|
|
40
40
|
agent
|
|
41
|
-
.on(:
|
|
42
|
-
.on(:
|
|
41
|
+
.on(:tool_start) { |_env, call| @calls << call }
|
|
42
|
+
.on(:llm_end) { |env| account(env[:metadata][:last_llm_usage]) }
|
|
43
43
|
.on(:faraday_error) { |_env, error| @failures << describe(error) }
|
|
44
44
|
.on(:open_router_server_error) { |_env, error| @failures << describe(error) }
|
|
45
45
|
.on(:standard_error) { |_env, error| @failures << describe(error) }
|
data/lib/brute/eval/world.rb
CHANGED
|
@@ -38,12 +38,12 @@ module Brute
|
|
|
38
38
|
kase.said
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
-
# :
|
|
41
|
+
# :tool_start is handed a mutable call env, and a :result set on it is
|
|
42
42
|
# answered without the tool ever running -- so a stub replaces the web,
|
|
43
43
|
# the calendar or the shell without the agent being built differently.
|
|
44
44
|
# A stub that answers to #call is handed the arguments.
|
|
45
45
|
def stub(agent, stubs)
|
|
46
|
-
agent.on(:
|
|
46
|
+
agent.on(:tool_start) do |_env, call|
|
|
47
47
|
canned = stubs[call[:name]]
|
|
48
48
|
|
|
49
49
|
unless canned.nil?
|
data/lib/brute/eval.rb
CHANGED
|
@@ -38,7 +38,7 @@ module Brute
|
|
|
38
38
|
# registry any other subscriber uses -- so the agent under evaluation is the
|
|
39
39
|
# agent that ships: no eval-only middleware, no branch in agent.ru. The
|
|
40
40
|
# model and the tool schemas are real; the tools themselves answer from the
|
|
41
|
-
# case's stubs, installed on :
|
|
41
|
+
# case's stubs, installed on :tool_start, which answers a call without
|
|
42
42
|
# executing it.
|
|
43
43
|
#
|
|
44
44
|
# Where a case wakes up is the World's business, and a deployment that
|
data/lib/brute/events/handler.rb
CHANGED
data/lib/brute/hooks.rb
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "bundler/setup"
|
|
4
|
+
require "delegate"
|
|
5
|
+
require "securerandom"
|
|
6
|
+
|
|
4
7
|
require "brute"
|
|
5
8
|
|
|
6
9
|
module Brute
|
|
@@ -9,87 +12,69 @@ module Brute
|
|
|
9
12
|
# Brute.agent
|
|
10
13
|
# .use(Brute::Middleware::MaxIterations)
|
|
11
14
|
# .run(->(env) { env[:messages].assistant("done") })
|
|
12
|
-
# .on(:
|
|
13
|
-
# .on(:
|
|
14
|
-
#
|
|
15
|
-
# Every subscriber is called with the turn env first, followed by whatever
|
|
16
|
-
# extras that event carries. A block that only wants the env can take one
|
|
17
|
-
# argument and ignore the rest.
|
|
18
|
-
#
|
|
19
|
-
# Emission points and payloads:
|
|
15
|
+
# .on(:llm_start) { |env| ... }
|
|
16
|
+
# .on(:tool_approve) { |_env, call| call[:name] != "exec" }
|
|
20
17
|
#
|
|
21
|
-
# :turn_start, :turn_end → the turn env (AgentPipeline#start; turn_end
|
|
22
|
-
# fires from an ensure, so it also fires on
|
|
23
|
-
# error)
|
|
24
|
-
# :turn_duration → env, started, finished: the turn's work is
|
|
25
|
-
# this event's block
|
|
26
|
-
# :middleware_added → an empty env, then the middleware and every
|
|
27
|
-
# argument `use` was given (fires at build time,
|
|
28
|
-
# so only subscribers registered before the `use`
|
|
29
|
-
# see it)
|
|
30
|
-
# :enter → env, the middleware instance, before the
|
|
31
|
-
# layer does anything
|
|
32
|
-
# :duration → env, started, finished, the middleware
|
|
33
|
-
# instance: the layer's work is this event's
|
|
34
|
-
# block, so it reports how long that took
|
|
35
|
-
# :exit → env, the middleware instance, marking the
|
|
36
|
-
# layer done (from an ensure, so it fires on
|
|
37
|
-
# error too)
|
|
38
|
-
# :before_llm, :after_llm → the turn env, around every LLM call
|
|
39
|
-
# :llm_duration → env, started, finished: the provider call is
|
|
40
|
-
# this event's block
|
|
41
|
-
# :llm_failure → the turn env, when the LLM call raises; the
|
|
42
|
-
# completion middleware then emits one of
|
|
43
|
-
# :faraday_error, :open_router_server_error or
|
|
44
|
-
# :standard_error with the exception as an extra
|
|
45
|
-
# :compact_duration → env, started, finished, the compactor: one
|
|
46
|
-
# strategy's attempt is this event's block, so
|
|
47
|
-
# the strategy that spends a model call is
|
|
48
|
-
# visible, and one that declined is still timed
|
|
49
|
-
# :compacted → env, {strategy:, before:, after:} — a
|
|
50
|
-
# compaction strategy rewrote env[:messages];
|
|
51
|
-
# what it replaced is already gone, so an
|
|
52
|
-
# application that keeps a transcript preserves
|
|
53
|
-
# it here
|
|
54
|
-
# :before_tool → env, call env {name:, arguments:, result:,
|
|
55
|
-
# denied:, events:, metadata:, turn_env:} —
|
|
56
|
-
# mutate :arguments to rewrite the call, or set
|
|
57
|
-
# :result to answer it without executing
|
|
58
|
-
# :approve_tool → env, call env — set :denied to true to deny
|
|
59
|
-
# the call, or to a String to deny it with that
|
|
60
|
-
# message
|
|
61
|
-
# :tool_duration → env, started, finished, call env: the tool's
|
|
62
|
-
# own execution is this event's block, so a
|
|
63
|
-
# skipped or denied call never fires it
|
|
64
|
-
# :after_tool → env, call env — mutate :result
|
|
65
|
-
#
|
|
66
|
-
# Subscribers run inline (tool events may fire from parallel threads).
|
|
67
|
-
# Exceptions propagate to the caller — layers that want fail-open semantics
|
|
68
|
-
# rescue in their own subscriber.
|
|
69
|
-
# Include this in anything that emits or subscribes and the event names are
|
|
70
|
-
# first class there: ENTER_EVENT rather than Brute::Hooks::ENTER_EVENT.
|
|
71
|
-
# The registry itself is Hooks::Registry, and Brute::Hooks.new builds one.
|
|
72
18
|
module Hooks
|
|
73
19
|
TURN_START_EVENT = :turn_start
|
|
74
20
|
TURN_DURATION_EVENT = :turn_duration
|
|
75
21
|
TURN_END_EVENT = :turn_end
|
|
22
|
+
TURN_FAILURE_EVENT = :turn_failure
|
|
76
23
|
MIDDLEWARE_ADDED_EVENT = :middleware_added
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
24
|
+
MIDDLEWARE_START_EVENT = :middleware_start
|
|
25
|
+
MIDDLEWARE_DURATION_EVENT = :middleware_duration
|
|
26
|
+
MIDDLEWARE_END_EVENT = :middleware_end
|
|
27
|
+
MIDDLEWARE_FAILURE_EVENT = :middleware_failure
|
|
28
|
+
LLM_START_EVENT = :llm_start
|
|
81
29
|
LLM_DURATION_EVENT = :llm_duration
|
|
82
|
-
|
|
30
|
+
LLM_END_EVENT = :llm_end
|
|
83
31
|
LLM_FAILURE_EVENT = :llm_failure
|
|
84
32
|
FARADAY_ERROR_EVENT = :faraday_error
|
|
85
33
|
OPEN_ROUTER_SERVER_ERROR_EVENT = :open_router_server_error
|
|
86
34
|
STANDARD_ERROR_EVENT = :standard_error
|
|
35
|
+
COMPACT_START_EVENT = :compact_start
|
|
87
36
|
COMPACT_DURATION_EVENT = :compact_duration
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
37
|
+
COMPACT_END_EVENT = :compact_end
|
|
38
|
+
COMPACT_FAILURE_EVENT = :compact_failure
|
|
39
|
+
CONTENT_EVENT = :content
|
|
40
|
+
REASONING_EVENT = :reasoning
|
|
41
|
+
TOOL_CALLS_EVENT = :tool_calls
|
|
42
|
+
TOOL_START_EVENT = :tool_start
|
|
43
|
+
TOOL_APPROVE_EVENT = :tool_approve
|
|
91
44
|
TOOL_DURATION_EVENT = :tool_duration
|
|
92
|
-
|
|
45
|
+
TOOL_END_EVENT = :tool_end
|
|
46
|
+
TOOL_FAILURE_EVENT = :tool_failure
|
|
47
|
+
|
|
48
|
+
# The env, wrapped, for as long as one call lasts. Everything emitted
|
|
49
|
+
# through it carries that call's id, and a call opened inside another
|
|
50
|
+
# delegates to it -- so the wrapper is the parent and unwrapping ends the
|
|
51
|
+
# call. It is still the env: SimpleDelegator forwards the rest.
|
|
52
|
+
class Trace < SimpleDelegator
|
|
53
|
+
attr_reader :id, :hooks
|
|
54
|
+
|
|
55
|
+
def initialize(env, hooks: nil, id: SecureRandom.uuid)
|
|
56
|
+
super(env)
|
|
57
|
+
if hooks
|
|
58
|
+
@hooks = hooks
|
|
59
|
+
elsif env.is_a?(Trace)
|
|
60
|
+
@hooks = env.hooks
|
|
61
|
+
end
|
|
62
|
+
@id = id
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def current_trace = self
|
|
66
|
+
|
|
67
|
+
def emit(event, *extras, &work) = @hooks.emit(
|
|
68
|
+
event,
|
|
69
|
+
self,
|
|
70
|
+
*extras,
|
|
71
|
+
@id,
|
|
72
|
+
&work
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
def emit_trace(&block) = self.class.new(self).tap(&block).then { |trace| trace.__getobj__ }
|
|
76
|
+
end
|
|
77
|
+
|
|
93
78
|
|
|
94
79
|
# The pub/sub registry a pipeline owns; `use` and `run` bind an emit to it.
|
|
95
80
|
class Registry
|
|
@@ -102,40 +87,33 @@ module Brute
|
|
|
102
87
|
self
|
|
103
88
|
end
|
|
104
89
|
|
|
105
|
-
# Fire an event. An emitter announces; it answers nothing, and what a
|
|
106
|
-
# subscriber's block happens to evaluate to is not a signal. A layer
|
|
107
|
-
# that wants to take part in a turn does it by mutating what it was
|
|
108
|
-
# handed, never by returning something.
|
|
109
|
-
#
|
|
110
|
-
# Given a block, the event is timed instead: the block is the work, and
|
|
111
|
-
# subscribers fire once it is done, called as
|
|
112
|
-
# `|env, started, finished, *extras|` rather than `|env, *extras|`.
|
|
113
|
-
# Both stamps are monotonic, so a clock adjustment mid-turn cannot
|
|
114
|
-
# produce a negative duration.
|
|
115
|
-
#
|
|
116
90
|
# The block is the work and nothing more: `emit` answers nothing in
|
|
117
91
|
# either form, so a caller that needs the work's value takes it inside
|
|
118
92
|
# the block.
|
|
119
93
|
#
|
|
120
94
|
# result = nil
|
|
121
|
-
# emit(
|
|
122
|
-
# # => .on(
|
|
95
|
+
# emit(MIDDLEWARE_DURATION_EVENT, env, self) { result = @app.call(env) }
|
|
96
|
+
# # => .on(MIDDLEWARE_DURATION_EVENT) { |env, started, finished, layer| ... }
|
|
123
97
|
#
|
|
124
98
|
# Subscribers fire from an ensure, so work that raises is still timed
|
|
125
99
|
# and still reported before the exception carries on up.
|
|
126
100
|
def emit(event, env, *extras, &block)
|
|
127
|
-
|
|
101
|
+
if block
|
|
102
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
103
|
+
|
|
104
|
+
begin
|
|
105
|
+
block.call
|
|
106
|
+
ensure
|
|
107
|
+
finished = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
108
|
+
@subscribers[event.to_sym].each { |subscriber| subscriber.call(
|
|
109
|
+
env,
|
|
110
|
+
started,
|
|
111
|
+
finished,
|
|
112
|
+
*extras,
|
|
113
|
+
) }
|
|
114
|
+
end
|
|
115
|
+
else
|
|
128
116
|
@subscribers[event.to_sym].each { |subscriber| subscriber.call(env, *extras) }
|
|
129
|
-
return nil
|
|
130
|
-
end
|
|
131
|
-
|
|
132
|
-
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
133
|
-
|
|
134
|
-
begin
|
|
135
|
-
block.call
|
|
136
|
-
ensure
|
|
137
|
-
finished = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
138
|
-
@subscribers[event.to_sym].each { |subscriber| subscriber.call(env, started, finished, *extras) }
|
|
139
117
|
end
|
|
140
118
|
|
|
141
119
|
nil
|
|
@@ -143,8 +121,6 @@ module Brute
|
|
|
143
121
|
|
|
144
122
|
def any?(event) = @subscribers[event.to_sym].any?
|
|
145
123
|
end
|
|
146
|
-
|
|
147
|
-
def self.new(...) = Registry.new(...)
|
|
148
124
|
end
|
|
149
125
|
end
|
|
150
126
|
|
|
@@ -152,38 +128,38 @@ __END__
|
|
|
152
128
|
|
|
153
129
|
describe "brute/hooks" do
|
|
154
130
|
it "emits to subscribers in registration order" do
|
|
155
|
-
hooks = Brute::Hooks.new
|
|
131
|
+
hooks = Brute::Hooks::Registry.new
|
|
156
132
|
seen = []
|
|
157
|
-
hooks.on(:
|
|
158
|
-
hooks.on(:
|
|
159
|
-
hooks.emit(:
|
|
133
|
+
hooks.on(:llm_start) { |env| seen << "a#{env}" }
|
|
134
|
+
hooks.on(:llm_start) { |env| seen << "b#{env}" }
|
|
135
|
+
hooks.emit(:llm_start, 1)
|
|
160
136
|
seen.should == ["a1", "b1"]
|
|
161
137
|
end
|
|
162
138
|
|
|
163
139
|
it "answers nothing: a subscriber takes part by mutating, not by returning" do
|
|
164
|
-
hooks = Brute::Hooks.new
|
|
165
|
-
hooks.on(:
|
|
140
|
+
hooks = Brute::Hooks::Registry.new
|
|
141
|
+
hooks.on(:tool_approve) { |_env, call| call[:denied] = true }
|
|
166
142
|
|
|
167
143
|
call = {}
|
|
168
|
-
hooks.emit(:
|
|
144
|
+
hooks.emit(:tool_approve, {}, call).should.be.nil
|
|
169
145
|
call[:denied].should.be.true
|
|
170
146
|
end
|
|
171
147
|
|
|
172
148
|
it "hands every subscriber the env first and the extras after" do
|
|
173
|
-
hooks = Brute::Hooks.new
|
|
149
|
+
hooks = Brute::Hooks::Registry.new
|
|
174
150
|
seen = []
|
|
175
|
-
hooks.on(:
|
|
176
|
-
hooks.emit(:
|
|
151
|
+
hooks.on(:tool_end) { |env, call| seen << [env, call] }
|
|
152
|
+
hooks.emit(:tool_end, :turn, :call)
|
|
177
153
|
seen.should == [[:turn, :call]]
|
|
178
154
|
end
|
|
179
155
|
|
|
180
156
|
it "times a block, hands subscribers start and finish, and reports even when the work raises" do
|
|
181
157
|
seen = []
|
|
182
|
-
hooks = Brute::Hooks.new
|
|
183
|
-
hooks.on(:
|
|
158
|
+
hooks = Brute::Hooks::Registry.new
|
|
159
|
+
hooks.on(:middleware_end) { |env, started, finished, subject| seen << [env, started, finished, subject] }
|
|
184
160
|
|
|
185
161
|
ran = nil
|
|
186
|
-
hooks.emit(:
|
|
162
|
+
hooks.emit(:middleware_end, :env, :layer) { ran = :work_result }.should.be.nil
|
|
187
163
|
ran.should == :work_result
|
|
188
164
|
seen.size.should == 1
|
|
189
165
|
env, started, finished, subject = seen.first
|
|
@@ -192,19 +168,19 @@ describe "brute/hooks" do
|
|
|
192
168
|
(finished - started).should.be >= 0
|
|
193
169
|
|
|
194
170
|
# Work that raises is still timed and still reported.
|
|
195
|
-
should.raise(RuntimeError) { hooks.emit(:
|
|
171
|
+
should.raise(RuntimeError) { hooks.emit(:middleware_end, :env, :layer) { raise "boom" } }
|
|
196
172
|
seen.size.should == 2
|
|
197
173
|
|
|
198
174
|
# Without a block it stays a point event: no timing argument.
|
|
199
175
|
args = []
|
|
200
|
-
hooks2 = Brute::Hooks.new
|
|
201
|
-
hooks2.on(:
|
|
202
|
-
hooks2.emit(:
|
|
176
|
+
hooks2 = Brute::Hooks::Registry.new
|
|
177
|
+
hooks2.on(:middleware_end) { |*received| args << received }
|
|
178
|
+
hooks2.emit(:middleware_end, :env, :layer)
|
|
203
179
|
args.should == [[:env, :layer]]
|
|
204
180
|
end
|
|
205
181
|
|
|
206
182
|
it "answers any? and stays chainable" do
|
|
207
|
-
hooks = Brute::Hooks.new
|
|
183
|
+
hooks = Brute::Hooks::Registry.new
|
|
208
184
|
hooks.any?(:turn_start).should.be.false
|
|
209
185
|
hooks.on(:turn_start) { nil }.should.equal?(hooks)
|
|
210
186
|
hooks.any?(:turn_start).should.be.true
|
|
@@ -55,7 +55,9 @@ module Brute
|
|
|
55
55
|
when :assistant
|
|
56
56
|
if message.tool_call?
|
|
57
57
|
blocks = []
|
|
58
|
-
|
|
58
|
+
unless message.content.to_s.empty?
|
|
59
|
+
blocks << { type: "text", text: message.content }
|
|
60
|
+
end
|
|
59
61
|
blocks += message.tool_calls.map { |tc| { type: "tool_use", id: tc.id, name: tc.name, input: tc.arguments } }
|
|
60
62
|
{ role: "assistant", content: blocks }
|
|
61
63
|
else
|
|
@@ -79,14 +81,22 @@ module Brute
|
|
|
79
81
|
|
|
80
82
|
text = blocks.select { |b| b.type == :text }.map(&:text).join
|
|
81
83
|
tool_calls = blocks.select { |b| b.type == :tool_use }.map do |b|
|
|
82
|
-
|
|
84
|
+
if b.input.respond_to?(:to_h)
|
|
85
|
+
arguments = b.input.to_h
|
|
86
|
+
else
|
|
87
|
+
arguments = b.input
|
|
88
|
+
end
|
|
83
89
|
Brute::ToolCall.new(id: b.id, name: b.name, arguments: arguments)
|
|
84
90
|
end
|
|
85
91
|
|
|
92
|
+
if tool_calls.empty?
|
|
93
|
+
tool_calls = nil
|
|
94
|
+
end
|
|
95
|
+
|
|
86
96
|
Brute::Message.new(
|
|
87
97
|
role: :assistant,
|
|
88
98
|
content: text,
|
|
89
|
-
tool_calls:
|
|
99
|
+
tool_calls: tool_calls,
|
|
90
100
|
)
|
|
91
101
|
end
|
|
92
102
|
end
|