brute 5.1.0 → 6.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/brute/compaction/middleware/sliding_window.rb +1 -1
- data/lib/brute/compaction/middleware/strategy.rb +0 -2
- data/lib/brute/compaction/summarize.rb +30 -21
- data/lib/brute/completion/lang_chain.rb +34 -31
- data/lib/brute/completion/llmrb.rb +31 -29
- data/lib/brute/completion/open_router.rb +43 -40
- data/lib/brute/completion/ruby_llm.rb +28 -27
- data/lib/brute/contrib/otel.rb +81 -49
- data/lib/brute/eval/case.rb +7 -2
- data/lib/brute/eval/suite.rb +10 -5
- data/lib/brute/eval/transcript.rb +4 -4
- data/lib/brute/eval/world.rb +2 -2
- data/lib/brute/eval.rb +1 -1
- data/lib/brute/events/handler.rb +3 -1
- data/lib/brute/events/prefixed_terminal_output.rb +3 -1
- data/lib/brute/events/terminal_output_handler.rb +1 -1
- data/lib/brute/hooks.rb +91 -112
- data/lib/brute/message_transport/anthropic.rb +13 -3
- data/lib/brute/message_transport/llm.rb +8 -4
- data/lib/brute/message_transport/open_router.rb +30 -16
- data/lib/brute/message_transport/openai.rb +14 -7
- data/lib/brute/message_transport/ruby_llm.rb +21 -17
- data/lib/brute/message_transport/ruby_open_ai.rb +59 -57
- data/lib/brute/messages.rb +15 -8
- data/lib/brute/middleware/000_base.rb +5 -5
- data/lib/brute/middleware/002_session_log.rb +12 -4
- data/lib/brute/middleware/008_checkpoint.rb +26 -18
- data/lib/brute/middleware/010_max_iterations.rb +1 -1
- data/lib/brute/middleware/020_system_prompt.rb +1 -1
- data/lib/brute/middleware/040_default_compaction_pipeline.rb +49 -45
- data/lib/brute/middleware/070_default_tool_pipeline.rb +44 -53
- data/lib/brute/prompt_template.rb +19 -15
- data/lib/brute/prompts/base.rb +19 -10
- data/lib/brute/prompts/environment.rb +3 -1
- data/lib/brute/prompts/instructions.rb +9 -7
- data/lib/brute/prompts/skills.rb +5 -3
- data/lib/brute/rack/adapter.rb +40 -23
- data/lib/brute/skill.rb +136 -89
- data/lib/brute/system_prompt.rb +9 -9
- data/lib/brute/token_counter.rb +3 -1
- data/lib/brute/tool.rb +10 -6
- data/lib/brute/tools/adapter.rb +53 -43
- data/lib/brute/tools/fs/snapshot_store.rb +5 -1
- data/lib/brute/tools/fs_patch.rb +16 -8
- data/lib/brute/tools/fs_read.rb +107 -80
- data/lib/brute/tools/fs_remove.rb +6 -2
- data/lib/brute/tools/fs_search.rb +14 -4
- data/lib/brute/tools/fs_undo.rb +6 -2
- data/lib/brute/tools/fs_write.rb +5 -1
- data/lib/brute/tools/net_fetch.rb +6 -2
- data/lib/brute/tools/question.rb +42 -39
- data/lib/brute/tools/shell.rb +20 -5
- data/lib/brute/tools/skill_load.rb +44 -39
- data/lib/brute/tools/todo_write.rb +19 -15
- data/lib/brute/truncation.rb +68 -43
- data/lib/brute/turn/agent_pipeline.rb +23 -13
- data/lib/brute/turn/compaction_pipeline.rb +4 -3
- data/lib/brute/turn/pipeline.rb +93 -96
- data/lib/brute/turn/tool_pipeline.rb +4 -3
- data/lib/brute/usage_detection/llmrb.rb +18 -14
- data/lib/brute/usage_detection/open_router.rb +20 -17
- data/lib/brute/usage_detection/ruby_llm.rb +18 -14
- data/lib/brute/usage_detection/usage.rb +10 -1
- data/lib/brute/utils/diff.rb +18 -10
- data/lib/brute/version.rb +1 -1
- data/lib/brute.rb +10 -11
- data/lib/brute_cli/providers/shell.rb +32 -29
- data/lib/brute_cli/providers/shell_response.rb +20 -18
- metadata +30 -5
- data/lib/brute/middleware/040_compaction_check.rb +0 -73
- data/lib/brute/middleware/070_tool_pipeline.rb +0 -61
- data/lib/brute/middleware/event_handler.rb +0 -27
data/lib/brute/contrib/otel.rb
CHANGED
|
@@ -19,8 +19,8 @@ module Brute
|
|
|
19
19
|
# Every event it needs already exists:
|
|
20
20
|
#
|
|
21
21
|
# turn_start / turn_end the span itself
|
|
22
|
-
#
|
|
23
|
-
#
|
|
22
|
+
# llm_end token usage, from env[:metadata][:last_llm_usage]
|
|
23
|
+
# tool_start / tool_end a span event per tool call and result
|
|
24
24
|
#
|
|
25
25
|
# The tracer is injectable; without one it asks OpenTelemetry, and when
|
|
26
26
|
# the SDK is not loaded `subscribe` does nothing at all.
|
|
@@ -35,33 +35,50 @@ module Brute
|
|
|
35
35
|
|
|
36
36
|
class << self
|
|
37
37
|
def subscribe(agent, tracer: default_tracer)
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
38
|
+
if tracer.nil?
|
|
39
|
+
agent
|
|
40
|
+
else
|
|
41
|
+
agent
|
|
42
|
+
.on(Brute::Hooks::TURN_START_EVENT) { |env| start(env, tracer) }
|
|
43
|
+
.on(Brute::Hooks::TURN_END_EVENT) { |env| finish(env) }
|
|
44
|
+
.on(Brute::Hooks::LLM_END_EVENT) { |env| record_usage(env) }
|
|
45
|
+
.on(Brute::Hooks::TOOL_START_EVENT) { |env, call| tool_called(env, call) }
|
|
46
|
+
.on(Brute::Hooks::TOOL_END_EVENT) { |env, call| tool_returned(env, call) }
|
|
47
|
+
.on(Brute::Hooks::LLM_FAILURE_EVENT) { |env| failed(env) }
|
|
48
|
+
.on(Brute::Hooks::MIDDLEWARE_DURATION_EVENT) { |env, started, finished, layer| layer_finished(
|
|
49
|
+
env,
|
|
50
|
+
started,
|
|
51
|
+
finished,
|
|
52
|
+
layer,
|
|
53
|
+
) }
|
|
54
|
+
.on(Brute::Hooks::TURN_DURATION_EVENT) { |env, started, finished| timed(env, "turn", finished - started) }
|
|
55
|
+
.on(Brute::Hooks::LLM_DURATION_EVENT) { |env, started, finished| timed(env, "llm", finished - started) }
|
|
56
|
+
.on(Brute::Hooks::TOOL_DURATION_EVENT) { |env, started, finished, call| timed(
|
|
57
|
+
env,
|
|
58
|
+
"tool",
|
|
59
|
+
finished - started,
|
|
60
|
+
"tool.name" => call[:name].to_s,
|
|
61
|
+
) }
|
|
62
|
+
end
|
|
51
63
|
end
|
|
52
64
|
|
|
53
65
|
def default_tracer
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
66
|
+
if defined?(::OpenTelemetry)
|
|
67
|
+
::OpenTelemetry.tracer_provider.tracer("brute", Brute::VERSION)
|
|
68
|
+
else
|
|
69
|
+
nil
|
|
70
|
+
end
|
|
57
71
|
end
|
|
58
72
|
|
|
59
73
|
private
|
|
60
74
|
|
|
61
75
|
def start(env, tracer)
|
|
62
|
-
env[:span] = tracer.start_span(
|
|
63
|
-
|
|
64
|
-
|
|
76
|
+
env[:span] = tracer.start_span(
|
|
77
|
+
SPAN_NAME,
|
|
78
|
+
attributes: {
|
|
79
|
+
"brute.messages" => env[:messages]&.size,
|
|
80
|
+
}.compact,
|
|
81
|
+
)
|
|
65
82
|
end
|
|
66
83
|
|
|
67
84
|
def finish(env)
|
|
@@ -73,40 +90,55 @@ module Brute
|
|
|
73
90
|
# arrives under the same names, and what it did not report is
|
|
74
91
|
# absent rather than zero.
|
|
75
92
|
def record_usage(env)
|
|
76
|
-
span
|
|
77
|
-
usage = env.dig(:metadata, :last_llm_usage)
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
93
|
+
span = env[:span]
|
|
94
|
+
usage = env.dig(:metadata, :last_llm_usage)
|
|
95
|
+
|
|
96
|
+
if span && usage
|
|
97
|
+
{
|
|
98
|
+
"gen_ai.usage.input_tokens" => usage.input,
|
|
99
|
+
"gen_ai.usage.output_tokens" => usage.output,
|
|
100
|
+
"gen_ai.usage.total_tokens" => usage.total,
|
|
101
|
+
"gen_ai.usage.reasoning_tokens" => usage.reasoning,
|
|
102
|
+
"gen_ai.usage.cache_read_tokens" => usage.cache_read,
|
|
103
|
+
"gen_ai.usage.cost" => usage.cost,
|
|
104
|
+
}.each do |name, value|
|
|
105
|
+
unless value.nil?
|
|
106
|
+
span.set_attribute(name, value)
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
87
110
|
end
|
|
88
111
|
|
|
89
112
|
def tool_called(env, call)
|
|
90
|
-
env[:span]&.add_event(
|
|
91
|
-
"
|
|
92
|
-
|
|
93
|
-
|
|
113
|
+
env[:span]&.add_event(
|
|
114
|
+
"tool_call",
|
|
115
|
+
attributes: {
|
|
116
|
+
"tool.name" => call[:name].to_s,
|
|
117
|
+
"tool.arguments" => call[:arguments].to_s,
|
|
118
|
+
},
|
|
119
|
+
)
|
|
94
120
|
end
|
|
95
121
|
|
|
96
122
|
def tool_returned(env, call)
|
|
97
|
-
env[:span]&.add_event(
|
|
98
|
-
"
|
|
99
|
-
|
|
100
|
-
|
|
123
|
+
env[:span]&.add_event(
|
|
124
|
+
"tool_result",
|
|
125
|
+
attributes: {
|
|
126
|
+
"tool.name" => call[:name].to_s,
|
|
127
|
+
"tool.bytes" => call[:result].to_s.bytesize,
|
|
128
|
+
},
|
|
129
|
+
)
|
|
101
130
|
end
|
|
102
131
|
|
|
103
|
-
# The layer's work is :
|
|
104
|
-
# with it rather than having to be correlated with :
|
|
132
|
+
# The layer's work is :middleware_duration's block, so start and finish arrive
|
|
133
|
+
# with it rather than having to be correlated with :middleware_start by hand.
|
|
105
134
|
def layer_finished(env, started, finished, layer)
|
|
106
|
-
env[:span]&.add_event(
|
|
107
|
-
"middleware
|
|
108
|
-
|
|
109
|
-
|
|
135
|
+
env[:span]&.add_event(
|
|
136
|
+
"middleware",
|
|
137
|
+
attributes: {
|
|
138
|
+
"middleware.name" => layer.class.name.to_s,
|
|
139
|
+
"middleware.duration" => finished - started,
|
|
140
|
+
},
|
|
141
|
+
)
|
|
110
142
|
end
|
|
111
143
|
|
|
112
144
|
# Every pair in the turn has a timed middle, so each one's duration
|
|
@@ -170,8 +202,8 @@ describe "brute/contrib/otel" do
|
|
|
170
202
|
tool_calls: [{ id: "tc1", name: "echo", arguments: { "text" => "hi" } }])
|
|
171
203
|
end
|
|
172
204
|
|
|
173
|
-
emit(Brute::Hooks::LLM_DURATION_EVENT
|
|
174
|
-
emit(Brute::Hooks::
|
|
205
|
+
env.emit(Brute::Hooks::LLM_DURATION_EVENT) { :provider_call }
|
|
206
|
+
env.emit(Brute::Hooks::LLM_END_EVENT)
|
|
175
207
|
env
|
|
176
208
|
end
|
|
177
209
|
end)
|
|
@@ -190,7 +222,7 @@ describe "brute/contrib/otel" do
|
|
|
190
222
|
span.events.map(&:first).should.include "tool_result"
|
|
191
223
|
span.events.find { |name, _| name == "tool_call" }.last["tool.name"].should == "echo"
|
|
192
224
|
|
|
193
|
-
# :
|
|
225
|
+
# :middleware_end is timed, so each layer reports its own duration.
|
|
194
226
|
layer_event = span.events.find { |name, _| name == "middleware" }
|
|
195
227
|
layer_event.last["middleware.name"].should == "Brute::Middleware::DefaultToolPipeline"
|
|
196
228
|
layer_event.last["middleware.duration"].should.be >= 0
|
data/lib/brute/eval/case.rb
CHANGED
|
@@ -8,7 +8,12 @@ module Brute
|
|
|
8
8
|
# The budget a case allows itself. Lenient on purpose: an agent that took
|
|
9
9
|
# one search too many is worth knowing about, but it is not the same
|
|
10
10
|
# fault as an agent that answered wrongly.
|
|
11
|
-
Budget = Data.define(
|
|
11
|
+
Budget = Data.define(
|
|
12
|
+
:iterations,
|
|
13
|
+
:tool_calls,
|
|
14
|
+
:tokens,
|
|
15
|
+
:seconds,
|
|
16
|
+
) do
|
|
12
17
|
def initialize(iterations: 10, tool_calls: 8, tokens: 100_000, seconds: 180)
|
|
13
18
|
super
|
|
14
19
|
end
|
|
@@ -94,7 +99,7 @@ module Brute
|
|
|
94
99
|
said_it(transcript),
|
|
95
100
|
afforded(transcript),
|
|
96
101
|
checked(transcript),
|
|
97
|
-
].flatten.compact.uniq
|
|
102
|
+
].flatten.compact.uniq,
|
|
98
103
|
)
|
|
99
104
|
end
|
|
100
105
|
|
data/lib/brute/eval/suite.rb
CHANGED
|
@@ -21,7 +21,12 @@ module Brute
|
|
|
21
21
|
#
|
|
22
22
|
# #run answers a process exit status, so an eval script ends `exit(...)`.
|
|
23
23
|
class Suite
|
|
24
|
-
Result = Data.define(
|
|
24
|
+
Result = Data.define(
|
|
25
|
+
:kase,
|
|
26
|
+
:run,
|
|
27
|
+
:transcript,
|
|
28
|
+
:verdict,
|
|
29
|
+
)
|
|
25
30
|
|
|
26
31
|
def initialize(agent:, cases:, world: World.new, out: $stdout)
|
|
27
32
|
@agent = agent
|
|
@@ -67,10 +72,10 @@ module Brute
|
|
|
67
72
|
transcript.published = @world.published.dup
|
|
68
73
|
|
|
69
74
|
Result.new(
|
|
70
|
-
kase:
|
|
71
|
-
run:
|
|
75
|
+
kase: kase,
|
|
76
|
+
run: run,
|
|
72
77
|
transcript: transcript,
|
|
73
|
-
verdict:
|
|
78
|
+
verdict: kase.verdict(transcript),
|
|
74
79
|
)
|
|
75
80
|
end
|
|
76
81
|
|
|
@@ -93,7 +98,7 @@ module Brute
|
|
|
93
98
|
" spent: #{transcript.iterations}/#{budget.iterations} iterations, " \
|
|
94
99
|
"#{transcript.calls.length}/#{budget.tool_calls} calls, " \
|
|
95
100
|
"#{transcript.tokens}/#{budget.tokens} tokens, " \
|
|
96
|
-
"#{transcript.seconds.round(1)}s"
|
|
101
|
+
"#{transcript.seconds.round(1)}s",
|
|
97
102
|
)
|
|
98
103
|
result.verdict.failures.each { |failure| @out.puts " - #{failure}" }
|
|
99
104
|
@out.puts " said:"
|
|
@@ -34,12 +34,12 @@ module Brute
|
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
# The call env the tool pipeline hands its subscribers is one mutable
|
|
37
|
-
# hash per call, so the entry kept here at :
|
|
37
|
+
# hash per call, so the entry kept here at :tool_start carries the
|
|
38
38
|
# result the tool answered with by the time anyone reads it.
|
|
39
39
|
def subscribe(agent)
|
|
40
40
|
agent
|
|
41
|
-
.on(:
|
|
42
|
-
.on(:
|
|
41
|
+
.on(:tool_start) { |_env, call| @calls << call }
|
|
42
|
+
.on(:llm_end) { |env| account(env[:metadata][:last_llm_usage]) }
|
|
43
43
|
.on(:faraday_error) { |_env, error| @failures << describe(error) }
|
|
44
44
|
.on(:open_router_server_error) { |_env, error| @failures << describe(error) }
|
|
45
45
|
.on(:standard_error) { |_env, error| @failures << describe(error) }
|
|
@@ -60,7 +60,7 @@ module Brute
|
|
|
60
60
|
if @env[:messages].nil?
|
|
61
61
|
""
|
|
62
62
|
else
|
|
63
|
-
@env.
|
|
63
|
+
@env.reply&.content.to_s
|
|
64
64
|
end
|
|
65
65
|
end
|
|
66
66
|
|
data/lib/brute/eval/world.rb
CHANGED
|
@@ -38,12 +38,12 @@ module Brute
|
|
|
38
38
|
kase.said
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
-
# :
|
|
41
|
+
# :tool_start is handed a mutable call env, and a :result set on it is
|
|
42
42
|
# answered without the tool ever running -- so a stub replaces the web,
|
|
43
43
|
# the calendar or the shell without the agent being built differently.
|
|
44
44
|
# A stub that answers to #call is handed the arguments.
|
|
45
45
|
def stub(agent, stubs)
|
|
46
|
-
agent.on(:
|
|
46
|
+
agent.on(:tool_start) do |_env, call|
|
|
47
47
|
canned = stubs[call[:name]]
|
|
48
48
|
|
|
49
49
|
unless canned.nil?
|
data/lib/brute/eval.rb
CHANGED
|
@@ -38,7 +38,7 @@ module Brute
|
|
|
38
38
|
# registry any other subscriber uses -- so the agent under evaluation is the
|
|
39
39
|
# agent that ships: no eval-only middleware, no branch in agent.ru. The
|
|
40
40
|
# model and the tool schemas are real; the tools themselves answer from the
|
|
41
|
-
# case's stubs, installed on :
|
|
41
|
+
# case's stubs, installed on :tool_start, which answers a call without
|
|
42
42
|
# executing it.
|
|
43
43
|
#
|
|
44
44
|
# Where a case wakes up is the World's business, and a deployment that
|
data/lib/brute/events/handler.rb
CHANGED
data/lib/brute/hooks.rb
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "bundler/setup"
|
|
4
|
+
require "delegate"
|
|
5
|
+
require "securerandom"
|
|
6
|
+
|
|
4
7
|
require "brute"
|
|
5
8
|
|
|
6
9
|
module Brute
|
|
@@ -9,87 +12,72 @@ module Brute
|
|
|
9
12
|
# Brute.agent
|
|
10
13
|
# .use(Brute::Middleware::MaxIterations)
|
|
11
14
|
# .run(->(env) { env[:messages].assistant("done") })
|
|
12
|
-
# .on(:
|
|
13
|
-
# .on(:
|
|
14
|
-
#
|
|
15
|
-
# Every subscriber is called with the turn env first, followed by whatever
|
|
16
|
-
# extras that event carries. A block that only wants the env can take one
|
|
17
|
-
# argument and ignore the rest.
|
|
18
|
-
#
|
|
19
|
-
# Emission points and payloads:
|
|
15
|
+
# .on(:llm_start) { |env| ... }
|
|
16
|
+
# .on(:tool_approve) { |_env, call| call[:name] != "exec" }
|
|
20
17
|
#
|
|
21
|
-
# :turn_start, :turn_end → the turn env (AgentPipeline#start; turn_end
|
|
22
|
-
# fires from an ensure, so it also fires on
|
|
23
|
-
# error)
|
|
24
|
-
# :turn_duration → env, started, finished: the turn's work is
|
|
25
|
-
# this event's block
|
|
26
|
-
# :middleware_added → an empty env, then the middleware and every
|
|
27
|
-
# argument `use` was given (fires at build time,
|
|
28
|
-
# so only subscribers registered before the `use`
|
|
29
|
-
# see it)
|
|
30
|
-
# :enter → env, the middleware instance, before the
|
|
31
|
-
# layer does anything
|
|
32
|
-
# :duration → env, started, finished, the middleware
|
|
33
|
-
# instance: the layer's work is this event's
|
|
34
|
-
# block, so it reports how long that took
|
|
35
|
-
# :exit → env, the middleware instance, marking the
|
|
36
|
-
# layer done (from an ensure, so it fires on
|
|
37
|
-
# error too)
|
|
38
|
-
# :before_llm, :after_llm → the turn env, around every LLM call
|
|
39
|
-
# :llm_duration → env, started, finished: the provider call is
|
|
40
|
-
# this event's block
|
|
41
|
-
# :llm_failure → the turn env, when the LLM call raises; the
|
|
42
|
-
# completion middleware then emits one of
|
|
43
|
-
# :faraday_error, :open_router_server_error or
|
|
44
|
-
# :standard_error with the exception as an extra
|
|
45
|
-
# :compact_duration → env, started, finished, the compactor: one
|
|
46
|
-
# strategy's attempt is this event's block, so
|
|
47
|
-
# the strategy that spends a model call is
|
|
48
|
-
# visible, and one that declined is still timed
|
|
49
|
-
# :compacted → env, {strategy:, before:, after:} — a
|
|
50
|
-
# compaction strategy rewrote env[:messages];
|
|
51
|
-
# what it replaced is already gone, so an
|
|
52
|
-
# application that keeps a transcript preserves
|
|
53
|
-
# it here
|
|
54
|
-
# :before_tool → env, call env {name:, arguments:, result:,
|
|
55
|
-
# denied:, events:, metadata:, turn_env:} —
|
|
56
|
-
# mutate :arguments to rewrite the call, or set
|
|
57
|
-
# :result to answer it without executing
|
|
58
|
-
# :approve_tool → env, call env — set :denied to true to deny
|
|
59
|
-
# the call, or to a String to deny it with that
|
|
60
|
-
# message
|
|
61
|
-
# :tool_duration → env, started, finished, call env: the tool's
|
|
62
|
-
# own execution is this event's block, so a
|
|
63
|
-
# skipped or denied call never fires it
|
|
64
|
-
# :after_tool → env, call env — mutate :result
|
|
65
|
-
#
|
|
66
|
-
# Subscribers run inline (tool events may fire from parallel threads).
|
|
67
|
-
# Exceptions propagate to the caller — layers that want fail-open semantics
|
|
68
|
-
# rescue in their own subscriber.
|
|
69
|
-
# Include this in anything that emits or subscribes and the event names are
|
|
70
|
-
# first class there: ENTER_EVENT rather than Brute::Hooks::ENTER_EVENT.
|
|
71
|
-
# The registry itself is Hooks::Registry, and Brute::Hooks.new builds one.
|
|
72
18
|
module Hooks
|
|
73
19
|
TURN_START_EVENT = :turn_start
|
|
74
20
|
TURN_DURATION_EVENT = :turn_duration
|
|
75
21
|
TURN_END_EVENT = :turn_end
|
|
22
|
+
TURN_FAILURE_EVENT = :turn_failure
|
|
76
23
|
MIDDLEWARE_ADDED_EVENT = :middleware_added
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
24
|
+
MIDDLEWARE_START_EVENT = :middleware_start
|
|
25
|
+
MIDDLEWARE_DURATION_EVENT = :middleware_duration
|
|
26
|
+
MIDDLEWARE_END_EVENT = :middleware_end
|
|
27
|
+
MIDDLEWARE_FAILURE_EVENT = :middleware_failure
|
|
28
|
+
LLM_START_EVENT = :llm_start
|
|
81
29
|
LLM_DURATION_EVENT = :llm_duration
|
|
82
|
-
|
|
30
|
+
LLM_END_EVENT = :llm_end
|
|
83
31
|
LLM_FAILURE_EVENT = :llm_failure
|
|
84
32
|
FARADAY_ERROR_EVENT = :faraday_error
|
|
85
33
|
OPEN_ROUTER_SERVER_ERROR_EVENT = :open_router_server_error
|
|
86
34
|
STANDARD_ERROR_EVENT = :standard_error
|
|
35
|
+
COMPACT_START_EVENT = :compact_start
|
|
87
36
|
COMPACT_DURATION_EVENT = :compact_duration
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
37
|
+
COMPACT_END_EVENT = :compact_end
|
|
38
|
+
COMPACT_FAILURE_EVENT = :compact_failure
|
|
39
|
+
CONTENT_EVENT = :content
|
|
40
|
+
REASONING_EVENT = :reasoning
|
|
41
|
+
TOOL_CALLS_EVENT = :tool_calls
|
|
42
|
+
TOOL_START_EVENT = :tool_start
|
|
43
|
+
TOOL_APPROVE_EVENT = :tool_approve
|
|
91
44
|
TOOL_DURATION_EVENT = :tool_duration
|
|
92
|
-
|
|
45
|
+
TOOL_END_EVENT = :tool_end
|
|
46
|
+
TOOL_FAILURE_EVENT = :tool_failure
|
|
47
|
+
|
|
48
|
+
# The env, wrapped, for as long as one call lasts. Everything emitted
|
|
49
|
+
# through it carries the trace itself as its last argument -- so a
|
|
50
|
+
# subscriber that wants the call's identity asks it for an id, and one
|
|
51
|
+
# that wants anything else about the call has it -- and a call opened
|
|
52
|
+
# inside another
|
|
53
|
+
# delegates to it -- so the wrapper is the parent and unwrapping ends the
|
|
54
|
+
# call. It is still the env: SimpleDelegator forwards the rest.
|
|
55
|
+
class Trace < SimpleDelegator
|
|
56
|
+
attr_reader :id, :hooks
|
|
57
|
+
|
|
58
|
+
def initialize(env, hooks: nil, id: SecureRandom.uuid)
|
|
59
|
+
super(env)
|
|
60
|
+
if hooks
|
|
61
|
+
@hooks = hooks
|
|
62
|
+
elsif env.is_a?(Trace)
|
|
63
|
+
@hooks = env.hooks
|
|
64
|
+
end
|
|
65
|
+
@id = id
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def current_trace = self
|
|
69
|
+
|
|
70
|
+
def emit(event, *extras, &work) = @hooks.emit(
|
|
71
|
+
event,
|
|
72
|
+
self,
|
|
73
|
+
*extras,
|
|
74
|
+
self,
|
|
75
|
+
&work
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
def emit_trace(&block) = self.class.new(self).tap(&block).then { |trace| trace.__getobj__ }
|
|
79
|
+
end
|
|
80
|
+
|
|
93
81
|
|
|
94
82
|
# The pub/sub registry a pipeline owns; `use` and `run` bind an emit to it.
|
|
95
83
|
class Registry
|
|
@@ -102,40 +90,33 @@ module Brute
|
|
|
102
90
|
self
|
|
103
91
|
end
|
|
104
92
|
|
|
105
|
-
# Fire an event. An emitter announces; it answers nothing, and what a
|
|
106
|
-
# subscriber's block happens to evaluate to is not a signal. A layer
|
|
107
|
-
# that wants to take part in a turn does it by mutating what it was
|
|
108
|
-
# handed, never by returning something.
|
|
109
|
-
#
|
|
110
|
-
# Given a block, the event is timed instead: the block is the work, and
|
|
111
|
-
# subscribers fire once it is done, called as
|
|
112
|
-
# `|env, started, finished, *extras|` rather than `|env, *extras|`.
|
|
113
|
-
# Both stamps are monotonic, so a clock adjustment mid-turn cannot
|
|
114
|
-
# produce a negative duration.
|
|
115
|
-
#
|
|
116
93
|
# The block is the work and nothing more: `emit` answers nothing in
|
|
117
94
|
# either form, so a caller that needs the work's value takes it inside
|
|
118
95
|
# the block.
|
|
119
96
|
#
|
|
120
97
|
# result = nil
|
|
121
|
-
# emit(
|
|
122
|
-
# # => .on(
|
|
98
|
+
# emit(MIDDLEWARE_DURATION_EVENT, env, self) { result = @app.call(env) }
|
|
99
|
+
# # => .on(MIDDLEWARE_DURATION_EVENT) { |env, started, finished, layer| ... }
|
|
123
100
|
#
|
|
124
101
|
# Subscribers fire from an ensure, so work that raises is still timed
|
|
125
102
|
# and still reported before the exception carries on up.
|
|
126
103
|
def emit(event, env, *extras, &block)
|
|
127
|
-
|
|
104
|
+
if block
|
|
105
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
106
|
+
|
|
107
|
+
begin
|
|
108
|
+
block.call
|
|
109
|
+
ensure
|
|
110
|
+
finished = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
111
|
+
@subscribers[event.to_sym].each { |subscriber| subscriber.call(
|
|
112
|
+
env,
|
|
113
|
+
started,
|
|
114
|
+
finished,
|
|
115
|
+
*extras,
|
|
116
|
+
) }
|
|
117
|
+
end
|
|
118
|
+
else
|
|
128
119
|
@subscribers[event.to_sym].each { |subscriber| subscriber.call(env, *extras) }
|
|
129
|
-
return nil
|
|
130
|
-
end
|
|
131
|
-
|
|
132
|
-
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
133
|
-
|
|
134
|
-
begin
|
|
135
|
-
block.call
|
|
136
|
-
ensure
|
|
137
|
-
finished = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
138
|
-
@subscribers[event.to_sym].each { |subscriber| subscriber.call(env, started, finished, *extras) }
|
|
139
120
|
end
|
|
140
121
|
|
|
141
122
|
nil
|
|
@@ -143,8 +124,6 @@ module Brute
|
|
|
143
124
|
|
|
144
125
|
def any?(event) = @subscribers[event.to_sym].any?
|
|
145
126
|
end
|
|
146
|
-
|
|
147
|
-
def self.new(...) = Registry.new(...)
|
|
148
127
|
end
|
|
149
128
|
end
|
|
150
129
|
|
|
@@ -152,38 +131,38 @@ __END__
|
|
|
152
131
|
|
|
153
132
|
describe "brute/hooks" do
|
|
154
133
|
it "emits to subscribers in registration order" do
|
|
155
|
-
hooks = Brute::Hooks.new
|
|
134
|
+
hooks = Brute::Hooks::Registry.new
|
|
156
135
|
seen = []
|
|
157
|
-
hooks.on(:
|
|
158
|
-
hooks.on(:
|
|
159
|
-
hooks.emit(:
|
|
136
|
+
hooks.on(:llm_start) { |env| seen << "a#{env}" }
|
|
137
|
+
hooks.on(:llm_start) { |env| seen << "b#{env}" }
|
|
138
|
+
hooks.emit(:llm_start, 1)
|
|
160
139
|
seen.should == ["a1", "b1"]
|
|
161
140
|
end
|
|
162
141
|
|
|
163
142
|
it "answers nothing: a subscriber takes part by mutating, not by returning" do
|
|
164
|
-
hooks = Brute::Hooks.new
|
|
165
|
-
hooks.on(:
|
|
143
|
+
hooks = Brute::Hooks::Registry.new
|
|
144
|
+
hooks.on(:tool_approve) { |_env, call| call[:denied] = true }
|
|
166
145
|
|
|
167
146
|
call = {}
|
|
168
|
-
hooks.emit(:
|
|
147
|
+
hooks.emit(:tool_approve, {}, call).should.be.nil
|
|
169
148
|
call[:denied].should.be.true
|
|
170
149
|
end
|
|
171
150
|
|
|
172
151
|
it "hands every subscriber the env first and the extras after" do
|
|
173
|
-
hooks = Brute::Hooks.new
|
|
152
|
+
hooks = Brute::Hooks::Registry.new
|
|
174
153
|
seen = []
|
|
175
|
-
hooks.on(:
|
|
176
|
-
hooks.emit(:
|
|
154
|
+
hooks.on(:tool_end) { |env, call| seen << [env, call] }
|
|
155
|
+
hooks.emit(:tool_end, :turn, :call)
|
|
177
156
|
seen.should == [[:turn, :call]]
|
|
178
157
|
end
|
|
179
158
|
|
|
180
159
|
it "times a block, hands subscribers start and finish, and reports even when the work raises" do
|
|
181
160
|
seen = []
|
|
182
|
-
hooks = Brute::Hooks.new
|
|
183
|
-
hooks.on(:
|
|
161
|
+
hooks = Brute::Hooks::Registry.new
|
|
162
|
+
hooks.on(:middleware_end) { |env, started, finished, subject| seen << [env, started, finished, subject] }
|
|
184
163
|
|
|
185
164
|
ran = nil
|
|
186
|
-
hooks.emit(:
|
|
165
|
+
hooks.emit(:middleware_end, :env, :layer) { ran = :work_result }.should.be.nil
|
|
187
166
|
ran.should == :work_result
|
|
188
167
|
seen.size.should == 1
|
|
189
168
|
env, started, finished, subject = seen.first
|
|
@@ -192,19 +171,19 @@ describe "brute/hooks" do
|
|
|
192
171
|
(finished - started).should.be >= 0
|
|
193
172
|
|
|
194
173
|
# Work that raises is still timed and still reported.
|
|
195
|
-
should.raise(RuntimeError) { hooks.emit(:
|
|
174
|
+
should.raise(RuntimeError) { hooks.emit(:middleware_end, :env, :layer) { raise "boom" } }
|
|
196
175
|
seen.size.should == 2
|
|
197
176
|
|
|
198
177
|
# Without a block it stays a point event: no timing argument.
|
|
199
178
|
args = []
|
|
200
|
-
hooks2 = Brute::Hooks.new
|
|
201
|
-
hooks2.on(:
|
|
202
|
-
hooks2.emit(:
|
|
179
|
+
hooks2 = Brute::Hooks::Registry.new
|
|
180
|
+
hooks2.on(:middleware_end) { |*received| args << received }
|
|
181
|
+
hooks2.emit(:middleware_end, :env, :layer)
|
|
203
182
|
args.should == [[:env, :layer]]
|
|
204
183
|
end
|
|
205
184
|
|
|
206
185
|
it "answers any? and stays chainable" do
|
|
207
|
-
hooks = Brute::Hooks.new
|
|
186
|
+
hooks = Brute::Hooks::Registry.new
|
|
208
187
|
hooks.any?(:turn_start).should.be.false
|
|
209
188
|
hooks.on(:turn_start) { nil }.should.equal?(hooks)
|
|
210
189
|
hooks.any?(:turn_start).should.be.true
|
|
@@ -55,7 +55,9 @@ module Brute
|
|
|
55
55
|
when :assistant
|
|
56
56
|
if message.tool_call?
|
|
57
57
|
blocks = []
|
|
58
|
-
|
|
58
|
+
unless message.content.to_s.empty?
|
|
59
|
+
blocks << { type: "text", text: message.content }
|
|
60
|
+
end
|
|
59
61
|
blocks += message.tool_calls.map { |tc| { type: "tool_use", id: tc.id, name: tc.name, input: tc.arguments } }
|
|
60
62
|
{ role: "assistant", content: blocks }
|
|
61
63
|
else
|
|
@@ -79,14 +81,22 @@ module Brute
|
|
|
79
81
|
|
|
80
82
|
text = blocks.select { |b| b.type == :text }.map(&:text).join
|
|
81
83
|
tool_calls = blocks.select { |b| b.type == :tool_use }.map do |b|
|
|
82
|
-
|
|
84
|
+
if b.input.respond_to?(:to_h)
|
|
85
|
+
arguments = b.input.to_h
|
|
86
|
+
else
|
|
87
|
+
arguments = b.input
|
|
88
|
+
end
|
|
83
89
|
Brute::ToolCall.new(id: b.id, name: b.name, arguments: arguments)
|
|
84
90
|
end
|
|
85
91
|
|
|
92
|
+
if tool_calls.empty?
|
|
93
|
+
tool_calls = nil
|
|
94
|
+
end
|
|
95
|
+
|
|
86
96
|
Brute::Message.new(
|
|
87
97
|
role: :assistant,
|
|
88
98
|
content: text,
|
|
89
|
-
tool_calls:
|
|
99
|
+
tool_calls: tool_calls,
|
|
90
100
|
)
|
|
91
101
|
end
|
|
92
102
|
end
|