braintrust 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 2e67bcb9e3752500ed9ac74f3ad5d46b7ee86867f648d772a4138b5491e3f405
|
|
4
|
+
data.tar.gz: 7949cd6453a74de3632a52704a13616ff620c85429f51bff1bb6405954ea9d9d
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 8b07cd2821efe35fc99235dfa5fde7c06ed3dd147262c30633e1469ac0d0bd30a92fcaabef301b19be5d27dda98542b3008c01c97d2d00e4bf35ee09996c602f
|
|
7
|
+
data.tar.gz: 7b8075fc99973163cd432253c5c9edd34e51e9b79fb4dd80dca7c8fd472716567acf4b68a6008be46545414c358b192ff180a4b07c76b722d0d691b69b219d47
|
data/README.md
CHANGED
|
@@ -601,7 +601,7 @@ See examples: [server/eval.ru](./examples/server/eval.ru),
|
|
|
601
601
|
|
|
602
602
|
## Documentation
|
|
603
603
|
|
|
604
|
-
- [Braintrust Documentation](https://
|
|
604
|
+
- [Braintrust Documentation](https://docs.braintrust.dev)
|
|
605
605
|
- [API Reference](https://gemdocs.org/gems/braintrust/)
|
|
606
606
|
|
|
607
607
|
## Troubleshooting
|
|
@@ -13,43 +13,76 @@ module Braintrust
|
|
|
13
13
|
module RubyLLM
|
|
14
14
|
module Instrumentation
|
|
15
15
|
# Chat instrumentation for RubyLLM.
|
|
16
|
-
# Wraps
|
|
16
|
+
# Wraps the method that makes one chat request, and execute_tool(), to create spans.
|
|
17
17
|
module Chat
|
|
18
18
|
def self.included(base)
|
|
19
19
|
# Guard against double-wrapping
|
|
20
|
-
|
|
20
|
+
return if applied?(base)
|
|
21
|
+
|
|
22
|
+
base.prepend(InstanceMethods)
|
|
23
|
+
base.prepend(base.public_method_defined?(:generate) ? GenerateTracing : CompleteTracing)
|
|
21
24
|
end
|
|
22
25
|
|
|
23
26
|
def self.applied?(base)
|
|
24
27
|
base.ancestors.include?(InstanceMethods)
|
|
25
28
|
end
|
|
26
29
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
30
|
+
# ruby_llm 1.x: each complete() call makes one request, and recursive calls from
|
|
31
|
+
# tool execution create nested spans.
|
|
32
|
+
module CompleteTracing
|
|
33
|
+
def complete(&block)
|
|
34
|
+
return block ? super : super() unless tracing_enabled?
|
|
35
|
+
|
|
36
|
+
trace_chat_request(block) { |wrapper| super(&wrapper) }
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# ruby_llm 2.x: complete() loops over step(), which calls generate() once per
|
|
41
|
+
# request, so each generate() call gets its own span. complete() gets a parent
|
|
42
|
+
# span so a turn's chat and tool spans share one trace when the caller has no span.
|
|
43
|
+
module GenerateTracing
|
|
44
|
+
def generate(&block)
|
|
45
|
+
return block ? super : super() unless tracing_enabled?
|
|
46
|
+
|
|
47
|
+
trace_chat_request(block) { |wrapper| super(&wrapper) }
|
|
48
|
+
end
|
|
49
|
+
|
|
31
50
|
def complete(&block)
|
|
32
51
|
return block ? super : super() unless tracing_enabled?
|
|
33
52
|
|
|
34
53
|
tracer = Braintrust::Contrib.tracer_for(self)
|
|
35
54
|
|
|
55
|
+
tracer.in_span("ruby_llm.complete") do |span|
|
|
56
|
+
Support::OTel.set_json_attr(span, "braintrust.span_attributes", {type: "task"})
|
|
57
|
+
input_messages = build_input_messages
|
|
58
|
+
Support::OTel.set_json_attr(span, "braintrust.input_json", input_messages) if input_messages.any?
|
|
59
|
+
|
|
60
|
+
response = block ? super : super()
|
|
61
|
+
|
|
62
|
+
# No metrics here: each request's tokens live on its ruby_llm.chat child span.
|
|
63
|
+
Support::OTel.set_json_attr(span, "braintrust.output_json", [format_message_for_input(response)]) if response
|
|
64
|
+
response
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
module InstanceMethods
|
|
70
|
+
private
|
|
71
|
+
|
|
72
|
+
# Traces one chat request. Yields the streaming wrapper block, or nil when not
|
|
73
|
+
# streaming, for the caller to pass to super.
|
|
74
|
+
def trace_chat_request(block)
|
|
75
|
+
tracer = Braintrust::Contrib.tracer_for(self)
|
|
76
|
+
|
|
36
77
|
tracer.in_span("ruby_llm.chat") do |span|
|
|
37
78
|
if block
|
|
38
|
-
|
|
39
|
-
handle_streaming_complete(span, block) do |&wrapper|
|
|
40
|
-
super(&wrapper)
|
|
41
|
-
end
|
|
79
|
+
handle_streaming_complete(span, block) { |&wrapper| yield wrapper }
|
|
42
80
|
else
|
|
43
|
-
|
|
44
|
-
handle_non_streaming_complete(span) do
|
|
45
|
-
super()
|
|
46
|
-
end
|
|
81
|
+
handle_non_streaming_complete(span) { yield nil }
|
|
47
82
|
end
|
|
48
83
|
end
|
|
49
84
|
end
|
|
50
85
|
|
|
51
|
-
private
|
|
52
|
-
|
|
53
86
|
# Wrap execute_tool() to trace tool executions.
|
|
54
87
|
# This is a private method in RubyLLM - wrapping it avoids
|
|
55
88
|
# conflicting with user-registered on_tool_call/on_tool_result callbacks.
|
|
@@ -199,22 +232,18 @@ module Braintrust
|
|
|
199
232
|
tool_schema = tool_params_schema(tool) ? build_basic_tool_schema(tool) : build_minimal_tool_schema(tool)
|
|
200
233
|
end
|
|
201
234
|
|
|
202
|
-
# Strip RubyLLM-specific fields to match native OpenAI format
|
|
235
|
+
# Strip RubyLLM-specific fields to match native OpenAI format. Build copies: the
|
|
236
|
+
# parameters hash is the tool's cached schema, which RubyLLM sends to the provider.
|
|
203
237
|
function_key = tool_schema&.key?(:function) ? :function : "function"
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
tool_params.delete("additionalProperties")
|
|
211
|
-
tool_params.delete(:additionalProperties)
|
|
212
|
-
params_key = tool_schema[function_key].key?(:parameters) ? :parameters : "parameters"
|
|
213
|
-
tool_schema[function_key][params_key] = tool_params
|
|
214
|
-
end
|
|
215
|
-
end
|
|
238
|
+
function = tool_schema && tool_schema[function_key]
|
|
239
|
+
return tool_schema unless function
|
|
240
|
+
|
|
241
|
+
params_key = function.key?(:parameters) ? :parameters : "parameters"
|
|
242
|
+
tool_params = function[params_key]
|
|
243
|
+
return tool_schema unless tool_params.is_a?(Hash)
|
|
216
244
|
|
|
217
|
-
|
|
245
|
+
stripped = tool_params.reject { |key, _| %w[strict additionalProperties].include?(key.to_s) }
|
|
246
|
+
tool_schema.merge(function_key => function.merge(params_key => stripped))
|
|
218
247
|
end
|
|
219
248
|
|
|
220
249
|
# A tool's JSON Schema, across ruby_llm versions.
|
|
@@ -388,7 +417,8 @@ module Braintrust
|
|
|
388
417
|
|
|
389
418
|
# Format tool calls into OpenAI format
|
|
390
419
|
def format_tool_calls(tool_calls)
|
|
391
|
-
|
|
420
|
+
# Both the pre-2.2 Hash and ToolCall::Collection expose values.
|
|
421
|
+
tool_calls.values.map do |tc|
|
|
392
422
|
args = tc.arguments
|
|
393
423
|
args_string = args.is_a?(String) ? args : JSON.generate(args)
|
|
394
424
|
|
|
@@ -423,20 +453,7 @@ module Braintrust
|
|
|
423
453
|
|
|
424
454
|
# Set metrics (token usage + time_to_first_token)
|
|
425
455
|
# RubyLLM stores tokens directly in the response hash, not in a nested usage object
|
|
426
|
-
metrics =
|
|
427
|
-
if result.respond_to?(:to_h)
|
|
428
|
-
result_hash = result.to_h
|
|
429
|
-
usage = {
|
|
430
|
-
"input_tokens" => result_hash[:input_tokens],
|
|
431
|
-
"output_tokens" => result_hash[:output_tokens],
|
|
432
|
-
"cached_tokens" => result_hash[:cached_tokens],
|
|
433
|
-
"cache_creation_tokens" => result_hash[:cache_creation_tokens]
|
|
434
|
-
}.compact
|
|
435
|
-
|
|
436
|
-
unless usage.empty?
|
|
437
|
-
metrics = Common.parse_usage_tokens(usage)
|
|
438
|
-
end
|
|
439
|
-
end
|
|
456
|
+
metrics = Common.message_usage_metrics(result)
|
|
440
457
|
metrics["time_to_first_token"] = time_to_first_token if time_to_first_token
|
|
441
458
|
Support::OTel.set_json_attr(span, "braintrust.metrics", metrics) unless metrics.empty?
|
|
442
459
|
end
|
|
@@ -479,20 +496,7 @@ module Braintrust
|
|
|
479
496
|
Support::OTel.set_json_attr(span, "braintrust.output_json", output)
|
|
480
497
|
|
|
481
498
|
# Set metrics (token usage + time_to_first_token)
|
|
482
|
-
metrics =
|
|
483
|
-
if response.respond_to?(:to_h)
|
|
484
|
-
response_hash = response.to_h
|
|
485
|
-
usage = {
|
|
486
|
-
"input_tokens" => response_hash[:input_tokens],
|
|
487
|
-
"output_tokens" => response_hash[:output_tokens],
|
|
488
|
-
"cached_tokens" => response_hash[:cached_tokens],
|
|
489
|
-
"cache_creation_tokens" => response_hash[:cache_creation_tokens]
|
|
490
|
-
}.compact
|
|
491
|
-
|
|
492
|
-
unless usage.empty?
|
|
493
|
-
metrics = Common.parse_usage_tokens(usage)
|
|
494
|
-
end
|
|
495
|
-
end
|
|
499
|
+
metrics = Common.message_usage_metrics(response)
|
|
496
500
|
metrics["time_to_first_token"] = time_to_first_token if time_to_first_token
|
|
497
501
|
Support::OTel.set_json_attr(span, "braintrust.metrics", metrics) unless metrics.empty?
|
|
498
502
|
end
|
|
@@ -6,6 +6,24 @@ module Braintrust
|
|
|
6
6
|
module Instrumentation
|
|
7
7
|
# Common utilities for RubyLLM instrumentation.
|
|
8
8
|
module Common
|
|
9
|
+
# Braintrust metrics for a RubyLLM message's token counts. ruby_llm 2.x renamed the cache
|
|
10
|
+
# counts from cached_tokens/cache_creation_tokens to cache_read_tokens/cache_write_tokens.
|
|
11
|
+
# @param message [Object] a RubyLLM message (anything responding to to_h)
|
|
12
|
+
# @return [Hash<String, Integer>] normalized metrics, empty when no counts are reported
|
|
13
|
+
def self.message_usage_metrics(message)
|
|
14
|
+
return {} unless message.respond_to?(:to_h)
|
|
15
|
+
|
|
16
|
+
counts = message.to_h
|
|
17
|
+
usage = {
|
|
18
|
+
"input_tokens" => counts[:input_tokens],
|
|
19
|
+
"output_tokens" => counts[:output_tokens],
|
|
20
|
+
"cached_tokens" => counts[:cached_tokens] || counts[:cache_read_tokens],
|
|
21
|
+
"cache_creation_tokens" => counts[:cache_creation_tokens] || counts[:cache_write_tokens]
|
|
22
|
+
}.compact
|
|
23
|
+
|
|
24
|
+
usage.empty? ? {} : parse_usage_tokens(usage)
|
|
25
|
+
end
|
|
26
|
+
|
|
9
27
|
# Parse RubyLLM usage tokens into normalized Braintrust metrics.
|
|
10
28
|
# RubyLLM normalizes token fields from all providers (OpenAI, Anthropic, etc.)
|
|
11
29
|
# into a consistent format:
|
|
@@ -7,7 +7,7 @@ module Braintrust
|
|
|
7
7
|
module Contrib
|
|
8
8
|
module RubyLLM
|
|
9
9
|
# Patcher for RubyLLM chat completions.
|
|
10
|
-
# Instruments RubyLLM::Chat#complete and #execute_tool
|
|
10
|
+
# Instruments RubyLLM::Chat#complete (plus #generate on 2.x) and #execute_tool.
|
|
11
11
|
class ChatPatcher < Braintrust::Contrib::Patcher
|
|
12
12
|
class << self
|
|
13
13
|
def applicable?
|
data/lib/braintrust/version.rb
CHANGED