braintrust 0.6.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '0681cdc4f7c2a0a17a634d61d6137413012c89d0c53683ae730725385d8cd37c'
4
- data.tar.gz: d5a143e99f067d34401d5e6eb06fbcdde51839094617428ed14a90600c2f5250
3
+ metadata.gz: 2e67bcb9e3752500ed9ac74f3ad5d46b7ee86867f648d772a4138b5491e3f405
4
+ data.tar.gz: 7949cd6453a74de3632a52704a13616ff620c85429f51bff1bb6405954ea9d9d
5
5
  SHA512:
6
- metadata.gz: 6fd5bdc6d41fb19d4af576a94bccd02fe0a8212d5e1c0367124bb81fa0e8079849e6eb4a5a4e0e43c170982f11318983fc65a51ff0dfad897835b7e1c40d099b
7
- data.tar.gz: 47d6e8ca51a4046549a55acef62c35f09e6e8f303d85d4c7cf706ed13a24dd50a88cd94c40fdcce445251362b4574efa33f646322767e278d11f520fc4ca81a1
6
+ metadata.gz: 8b07cd2821efe35fc99235dfa5fde7c06ed3dd147262c30633e1469ac0d0bd30a92fcaabef301b19be5d27dda98542b3008c01c97d2d00e4bf35ee09996c602f
7
+ data.tar.gz: 7b8075fc99973163cd432253c5c9edd34e51e9b79fb4dd80dca7c8fd472716567acf4b68a6008be46545414c358b192ff180a4b07c76b722d0d691b69b219d47
data/README.md CHANGED
@@ -601,7 +601,7 @@ See examples: [server/eval.ru](./examples/server/eval.ru),
601
601
 
602
602
  ## Documentation
603
603
 
604
- - [Braintrust Documentation](https://www.braintrust.dev/docs)
604
+ - [Braintrust Documentation](https://docs.braintrust.dev)
605
605
  - [API Reference](https://gemdocs.org/gems/braintrust/)
606
606
 
607
607
  ## Troubleshooting
@@ -13,43 +13,76 @@ module Braintrust
13
13
  module RubyLLM
14
14
  module Instrumentation
15
15
  # Chat instrumentation for RubyLLM.
16
- # Wraps complete() and execute_tool() methods to create spans.
16
+ # Wraps the method that makes one chat request, and execute_tool(), to create spans.
17
17
  module Chat
18
18
  def self.included(base)
19
19
  # Guard against double-wrapping
20
- base.prepend(InstanceMethods) unless applied?(base)
20
+ return if applied?(base)
21
+
22
+ base.prepend(InstanceMethods)
23
+ base.prepend(base.public_method_defined?(:generate) ? GenerateTracing : CompleteTracing)
21
24
  end
22
25
 
23
26
  def self.applied?(base)
24
27
  base.ancestors.include?(InstanceMethods)
25
28
  end
26
29
 
27
- module InstanceMethods
28
- # Wrap complete() to trace chat completions.
29
- # Each call creates a span - recursive calls from tool execution
30
- # create nested spans (each is a separate API call).
30
+ # ruby_llm 1.x: each complete() call makes one request, and recursive calls from
31
+ # tool execution create nested spans.
32
+ module CompleteTracing
33
+ def complete(&block)
34
+ return block ? super : super() unless tracing_enabled?
35
+
36
+ trace_chat_request(block) { |wrapper| super(&wrapper) }
37
+ end
38
+ end
39
+
40
+ # ruby_llm 2.x: complete() loops over step(), which calls generate() once per
41
+ # request, so each generate() call gets its own span. complete() gets a parent
42
+ # span so a turn's chat and tool spans share one trace when the caller has no span.
43
+ module GenerateTracing
44
+ def generate(&block)
45
+ return block ? super : super() unless tracing_enabled?
46
+
47
+ trace_chat_request(block) { |wrapper| super(&wrapper) }
48
+ end
49
+
31
50
  def complete(&block)
32
51
  return block ? super : super() unless tracing_enabled?
33
52
 
34
53
  tracer = Braintrust::Contrib.tracer_for(self)
35
54
 
55
+ tracer.in_span("ruby_llm.complete") do |span|
56
+ Support::OTel.set_json_attr(span, "braintrust.span_attributes", {type: "task"})
57
+ input_messages = build_input_messages
58
+ Support::OTel.set_json_attr(span, "braintrust.input_json", input_messages) if input_messages.any?
59
+
60
+ response = block ? super : super()
61
+
62
+ # No metrics here: each request's tokens live on its ruby_llm.chat child span.
63
+ Support::OTel.set_json_attr(span, "braintrust.output_json", [format_message_for_input(response)]) if response
64
+ response
65
+ end
66
+ end
67
+ end
68
+
69
+ module InstanceMethods
70
+ private
71
+
72
+ # Traces one chat request. Yields the streaming wrapper block, or nil when not
73
+ # streaming, for the caller to pass to super.
74
+ def trace_chat_request(block)
75
+ tracer = Braintrust::Contrib.tracer_for(self)
76
+
36
77
  tracer.in_span("ruby_llm.chat") do |span|
37
78
  if block
38
- # Streaming: pass a block that calls super() with the wrapper
39
- handle_streaming_complete(span, block) do |&wrapper|
40
- super(&wrapper)
41
- end
79
+ handle_streaming_complete(span, block) { |&wrapper| yield wrapper }
42
80
  else
43
- # Non-streaming: pass a block that calls super()
44
- handle_non_streaming_complete(span) do
45
- super()
46
- end
81
+ handle_non_streaming_complete(span) { yield nil }
47
82
  end
48
83
  end
49
84
  end
50
85
 
51
- private
52
-
53
86
  # Wrap execute_tool() to trace tool executions.
54
87
  # This is a private method in RubyLLM - wrapping it avoids
55
88
  # conflicting with user-registered on_tool_call/on_tool_result callbacks.
@@ -199,22 +232,18 @@ module Braintrust
199
232
  tool_schema = tool_params_schema(tool) ? build_basic_tool_schema(tool) : build_minimal_tool_schema(tool)
200
233
  end
201
234
 
202
- # Strip RubyLLM-specific fields to match native OpenAI format
235
+ # Strip RubyLLM-specific fields to match native OpenAI format. Build copies: the
236
+ # parameters hash is the tool's cached schema, which RubyLLM sends to the provider.
203
237
  function_key = tool_schema&.key?(:function) ? :function : "function"
204
- if tool_schema && tool_schema[function_key]
205
- tool_params = tool_schema[function_key][:parameters] || tool_schema[function_key]["parameters"]
206
- if tool_params.is_a?(Hash)
207
- tool_params = tool_params.dup if tool_params.frozen?
208
- tool_params.delete("strict")
209
- tool_params.delete(:strict)
210
- tool_params.delete("additionalProperties")
211
- tool_params.delete(:additionalProperties)
212
- params_key = tool_schema[function_key].key?(:parameters) ? :parameters : "parameters"
213
- tool_schema[function_key][params_key] = tool_params
214
- end
215
- end
238
+ function = tool_schema && tool_schema[function_key]
239
+ return tool_schema unless function
240
+
241
+ params_key = function.key?(:parameters) ? :parameters : "parameters"
242
+ tool_params = function[params_key]
243
+ return tool_schema unless tool_params.is_a?(Hash)
216
244
 
217
- tool_schema
245
+ stripped = tool_params.reject { |key, _| %w[strict additionalProperties].include?(key.to_s) }
246
+ tool_schema.merge(function_key => function.merge(params_key => stripped))
218
247
  end
219
248
 
220
249
  # A tool's JSON Schema, across ruby_llm versions.
@@ -388,7 +417,8 @@ module Braintrust
388
417
 
389
418
  # Format tool calls into OpenAI format
390
419
  def format_tool_calls(tool_calls)
391
- tool_calls.map do |_id, tc|
420
+ # Both the pre-2.2 Hash and ToolCall::Collection expose values.
421
+ tool_calls.values.map do |tc|
392
422
  args = tc.arguments
393
423
  args_string = args.is_a?(String) ? args : JSON.generate(args)
394
424
 
@@ -423,20 +453,7 @@ module Braintrust
423
453
 
424
454
  # Set metrics (token usage + time_to_first_token)
425
455
  # RubyLLM stores tokens directly in the response hash, not in a nested usage object
426
- metrics = {}
427
- if result.respond_to?(:to_h)
428
- result_hash = result.to_h
429
- usage = {
430
- "input_tokens" => result_hash[:input_tokens],
431
- "output_tokens" => result_hash[:output_tokens],
432
- "cached_tokens" => result_hash[:cached_tokens],
433
- "cache_creation_tokens" => result_hash[:cache_creation_tokens]
434
- }.compact
435
-
436
- unless usage.empty?
437
- metrics = Common.parse_usage_tokens(usage)
438
- end
439
- end
456
+ metrics = Common.message_usage_metrics(result)
440
457
  metrics["time_to_first_token"] = time_to_first_token if time_to_first_token
441
458
  Support::OTel.set_json_attr(span, "braintrust.metrics", metrics) unless metrics.empty?
442
459
  end
@@ -479,20 +496,7 @@ module Braintrust
479
496
  Support::OTel.set_json_attr(span, "braintrust.output_json", output)
480
497
 
481
498
  # Set metrics (token usage + time_to_first_token)
482
- metrics = {}
483
- if response.respond_to?(:to_h)
484
- response_hash = response.to_h
485
- usage = {
486
- "input_tokens" => response_hash[:input_tokens],
487
- "output_tokens" => response_hash[:output_tokens],
488
- "cached_tokens" => response_hash[:cached_tokens],
489
- "cache_creation_tokens" => response_hash[:cache_creation_tokens]
490
- }.compact
491
-
492
- unless usage.empty?
493
- metrics = Common.parse_usage_tokens(usage)
494
- end
495
- end
499
+ metrics = Common.message_usage_metrics(response)
496
500
  metrics["time_to_first_token"] = time_to_first_token if time_to_first_token
497
501
  Support::OTel.set_json_attr(span, "braintrust.metrics", metrics) unless metrics.empty?
498
502
  end
@@ -6,6 +6,24 @@ module Braintrust
6
6
  module Instrumentation
7
7
  # Common utilities for RubyLLM instrumentation.
8
8
  module Common
9
+ # Braintrust metrics for a RubyLLM message's token counts. ruby_llm 2.x renamed the cache
10
+ # counts from cached_tokens/cache_creation_tokens to cache_read_tokens/cache_write_tokens.
11
+ # @param message [Object] a RubyLLM message (anything responding to to_h)
12
+ # @return [Hash<String, Integer>] normalized metrics, empty when no counts are reported
13
+ def self.message_usage_metrics(message)
14
+ return {} unless message.respond_to?(:to_h)
15
+
16
+ counts = message.to_h
17
+ usage = {
18
+ "input_tokens" => counts[:input_tokens],
19
+ "output_tokens" => counts[:output_tokens],
20
+ "cached_tokens" => counts[:cached_tokens] || counts[:cache_read_tokens],
21
+ "cache_creation_tokens" => counts[:cache_creation_tokens] || counts[:cache_write_tokens]
22
+ }.compact
23
+
24
+ usage.empty? ? {} : parse_usage_tokens(usage)
25
+ end
26
+
9
27
  # Parse RubyLLM usage tokens into normalized Braintrust metrics.
10
28
  # RubyLLM normalizes token fields from all providers (OpenAI, Anthropic, etc.)
11
29
  # into a consistent format:
@@ -7,7 +7,7 @@ module Braintrust
7
7
  module Contrib
8
8
  module RubyLLM
9
9
  # Patcher for RubyLLM chat completions.
10
- # Instruments RubyLLM::Chat#complete and #execute_tool methods.
10
+ # Instruments RubyLLM::Chat#complete (plus #generate on 2.x) and #execute_tool.
11
11
  class ChatPatcher < Braintrust::Contrib::Patcher
12
12
  class << self
13
13
  def applicable?
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Braintrust
4
- VERSION = "0.6.0"
4
+ VERSION = "0.6.1"
5
5
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: braintrust
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.0
4
+ version: 0.6.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Braintrust