braintrust 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/braintrust/trace/contrib/anthropic.rb +82 -156
- data/lib/braintrust/trace/contrib/github.com/alexrudall/ruby-openai/ruby-openai.rb +141 -0
- data/lib/braintrust/trace/contrib/openai.rb +118 -5
- data/lib/braintrust/trace.rb +21 -3
- data/lib/braintrust/version.rb +1 -1
- metadata +4 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6321acf7b780922ed97ea3cc57dde47a52947a10650a082dcfd9af780056d99a
|
|
4
|
+
data.tar.gz: 67c181e53537829931de704c7503cc056646652f9c1a61d914bc1ee0b7af69a2
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: bb8546fdbf0a448016a1d31ceb8729a40be59e0d8d081ef275f763a11dbb2f5df0134ec52fc3b1c15c41d9dcdf42fbbe6becaf00ab7ac882c8f2f7e173a9a61f
|
|
7
|
+
data.tar.gz: 41e6d13504302a3b3ec26697cb50ce4736040d910c8e293d37088311922daa77274fc4214c86543ca21209bf56e82b2f6f00d5fbc2d7d0d0baad6f1e77cc48ff
|
|
@@ -196,7 +196,6 @@ module Braintrust
|
|
|
196
196
|
wrapper = Module.new do
|
|
197
197
|
define_method(:stream) do |**params, &block|
|
|
198
198
|
tracer = tracer_provider.tracer("braintrust")
|
|
199
|
-
aggregated_events = []
|
|
200
199
|
|
|
201
200
|
metadata = {
|
|
202
201
|
"provider" => "anthropic",
|
|
@@ -256,183 +255,110 @@ module Braintrust
|
|
|
256
255
|
end
|
|
257
256
|
|
|
258
257
|
# Store references on the stream object itself for the wrapper
|
|
259
|
-
stream.instance_variable_set(:@braintrust_aggregated_events, aggregated_events)
|
|
260
258
|
stream.instance_variable_set(:@braintrust_span, span)
|
|
261
259
|
stream.instance_variable_set(:@braintrust_metadata, metadata)
|
|
260
|
+
stream.instance_variable_set(:@braintrust_span_finished, false)
|
|
262
261
|
|
|
263
|
-
#
|
|
264
|
-
|
|
265
|
-
stream.define_singleton_method(:each) do |&user_block|
|
|
266
|
-
events = instance_variable_get(:@braintrust_aggregated_events)
|
|
267
|
-
span_obj = instance_variable_get(:@braintrust_span)
|
|
268
|
-
meta = instance_variable_get(:@braintrust_metadata)
|
|
269
|
-
|
|
270
|
-
begin
|
|
271
|
-
original_each.call do |event|
|
|
272
|
-
# Store event data for aggregation
|
|
273
|
-
events << event.to_h if event.respond_to?(:to_h)
|
|
274
|
-
# Call user's block if provided
|
|
275
|
-
user_block&.call(event)
|
|
276
|
-
end
|
|
277
|
-
rescue => e
|
|
278
|
-
span_obj.record_exception(e)
|
|
279
|
-
span_obj.status = ::OpenTelemetry::Trace::Status.error("Streaming error: #{e.message}")
|
|
280
|
-
raise
|
|
281
|
-
ensure
|
|
282
|
-
# Always aggregate and finish span after stream completes
|
|
283
|
-
unless events.empty?
|
|
284
|
-
aggregated_output = Braintrust::Trace::Anthropic.aggregate_streaming_events(events)
|
|
285
|
-
|
|
286
|
-
# Set output
|
|
287
|
-
if aggregated_output[:content]
|
|
288
|
-
output = [{
|
|
289
|
-
role: "assistant",
|
|
290
|
-
content: aggregated_output[:content]
|
|
291
|
-
}]
|
|
292
|
-
Braintrust::Trace::Anthropic.set_json_attr(span_obj, "braintrust.output_json", output)
|
|
293
|
-
end
|
|
294
|
-
|
|
295
|
-
# Set metrics if usage is available
|
|
296
|
-
if aggregated_output[:usage]
|
|
297
|
-
metrics = Braintrust::Trace::Anthropic.parse_usage_tokens(aggregated_output[:usage])
|
|
298
|
-
Braintrust::Trace::Anthropic.set_json_attr(span_obj, "braintrust.metrics", metrics) unless metrics.empty?
|
|
299
|
-
end
|
|
300
|
-
|
|
301
|
-
# Update metadata with response fields
|
|
302
|
-
meta["stop_reason"] = aggregated_output[:stop_reason] if aggregated_output[:stop_reason]
|
|
303
|
-
meta["model"] = aggregated_output[:model] if aggregated_output[:model]
|
|
304
|
-
Braintrust::Trace::Anthropic.set_json_attr(span_obj, "braintrust.metadata", meta)
|
|
305
|
-
end
|
|
262
|
+
# Local helper for brevity
|
|
263
|
+
set_json_attr = ->(attr_name, obj) { Braintrust::Trace::Anthropic.set_json_attr(span, attr_name, obj) }
|
|
306
264
|
|
|
307
|
-
|
|
265
|
+
# Helper lambda to extract stream data and set span attributes
|
|
266
|
+
# This is DRY - used by both .each() and .text() wrappers
|
|
267
|
+
extract_stream_metadata = lambda do
|
|
268
|
+
# Extract the SDK's internal accumulated message (built during streaming)
|
|
269
|
+
acc_msg = stream.instance_variable_get(:@accumated_message_snapshot)
|
|
270
|
+
return unless acc_msg
|
|
271
|
+
|
|
272
|
+
# Set output from accumulated message
|
|
273
|
+
if acc_msg.respond_to?(:content) && acc_msg.content
|
|
274
|
+
content_array = acc_msg.content.map(&:to_h)
|
|
275
|
+
output = [{
|
|
276
|
+
role: acc_msg.respond_to?(:role) ? acc_msg.role : "assistant",
|
|
277
|
+
content: content_array
|
|
278
|
+
}]
|
|
279
|
+
set_json_attr.call("braintrust.output_json", output)
|
|
308
280
|
end
|
|
309
|
-
end
|
|
310
281
|
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
282
|
+
# Set metrics from accumulated message
|
|
283
|
+
if acc_msg.respond_to?(:usage) && acc_msg.usage
|
|
284
|
+
metrics = Braintrust::Trace::Anthropic.parse_usage_tokens(acc_msg.usage)
|
|
285
|
+
set_json_attr.call("braintrust.metrics", metrics) unless metrics.empty?
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
# Update metadata with response fields
|
|
289
|
+
if acc_msg.respond_to?(:stop_reason) && acc_msg.stop_reason
|
|
290
|
+
metadata["stop_reason"] = acc_msg.stop_reason
|
|
291
|
+
end
|
|
292
|
+
if acc_msg.respond_to?(:model) && acc_msg.model
|
|
293
|
+
metadata["model"] = acc_msg.model
|
|
294
|
+
end
|
|
295
|
+
set_json_attr.call("braintrust.metadata", metadata)
|
|
314
296
|
end
|
|
315
297
|
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
298
|
+
# Helper lambda to finish span (prevents double-finishing via closure)
|
|
299
|
+
finish_braintrust_span = lambda do
|
|
300
|
+
return if stream.instance_variable_get(:@braintrust_span_finished)
|
|
301
|
+
stream.instance_variable_set(:@braintrust_span_finished, true)
|
|
319
302
|
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
303
|
+
extract_stream_metadata.call
|
|
304
|
+
span.finish
|
|
305
|
+
end
|
|
323
306
|
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
# Track content blocks by index
|
|
338
|
-
content_blocks = {}
|
|
339
|
-
content_builders = {}
|
|
340
|
-
|
|
341
|
-
events.each do |event|
|
|
342
|
-
event_type = event[:type] || event["type"]
|
|
343
|
-
next unless event_type
|
|
344
|
-
|
|
345
|
-
case event_type
|
|
346
|
-
when "message_start"
|
|
347
|
-
# Extract model and initial usage (input tokens, cache tokens)
|
|
348
|
-
message = event[:message] || event["message"]
|
|
349
|
-
if message
|
|
350
|
-
result[:model] = message[:model] || message["model"]
|
|
351
|
-
if message[:usage] || message["usage"]
|
|
352
|
-
usage = message[:usage] || message["usage"]
|
|
353
|
-
result[:usage].merge!(usage)
|
|
354
|
-
end
|
|
307
|
+
# Wrap .each() to ensure span finishes after consumption
|
|
308
|
+
original_each = stream.method(:each)
|
|
309
|
+
stream.define_singleton_method(:each) do |&user_block|
|
|
310
|
+
# Consume stream, calling user's block for each event
|
|
311
|
+
# The SDK builds @accumated_message_snapshot internally
|
|
312
|
+
original_each.call(&user_block)
|
|
313
|
+
rescue => e
|
|
314
|
+
span.record_exception(e)
|
|
315
|
+
span.status = ::OpenTelemetry::Trace::Status.error("Streaming error: #{e.message}")
|
|
316
|
+
raise
|
|
317
|
+
ensure
|
|
318
|
+
# Extract accumulated message and finish span
|
|
319
|
+
finish_braintrust_span.call
|
|
355
320
|
end
|
|
356
321
|
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
content_blocks[index] = content_block if index && content_block
|
|
362
|
-
|
|
363
|
-
when "content_block_delta"
|
|
364
|
-
# Accumulate deltas for content blocks
|
|
365
|
-
index = event[:index] || event["index"]
|
|
366
|
-
delta = event[:delta] || event["delta"]
|
|
367
|
-
next unless index && delta
|
|
368
|
-
|
|
369
|
-
delta_type = delta[:type] || delta["type"]
|
|
370
|
-
content_blocks[index] ||= {}
|
|
371
|
-
|
|
372
|
-
case delta_type
|
|
373
|
-
when "text_delta"
|
|
374
|
-
# Accumulate text
|
|
375
|
-
text = delta[:text] || delta["text"]
|
|
376
|
-
if text
|
|
377
|
-
content_builders[index] ||= ""
|
|
378
|
-
content_builders[index] += text
|
|
379
|
-
content_blocks[index][:type] = "text"
|
|
380
|
-
content_blocks[index]["type"] = "text"
|
|
381
|
-
end
|
|
322
|
+
# Wrap .text() to return an Enumerable that ensures span finishes
|
|
323
|
+
original_text = stream.method(:text)
|
|
324
|
+
stream.define_singleton_method(:text) do
|
|
325
|
+
text_enum = original_text.call
|
|
382
326
|
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
327
|
+
# Return wrapper Enumerable that finishes span after consumption
|
|
328
|
+
Enumerator.new do |y|
|
|
329
|
+
# Consume text enumerable (this consumes underlying stream)
|
|
330
|
+
# The SDK builds @accumated_message_snapshot internally
|
|
331
|
+
text_enum.each { |text| y << text }
|
|
332
|
+
rescue => e
|
|
333
|
+
span.record_exception(e)
|
|
334
|
+
span.status = ::OpenTelemetry::Trace::Status.error("Streaming error: #{e.message}")
|
|
335
|
+
raise
|
|
336
|
+
ensure
|
|
337
|
+
# Extract accumulated message and finish span
|
|
338
|
+
finish_braintrust_span.call
|
|
391
339
|
end
|
|
392
340
|
end
|
|
393
341
|
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
342
|
+
# Wrap .close() to ensure span finishes even if stream not consumed
|
|
343
|
+
original_close = stream.method(:close)
|
|
344
|
+
stream.define_singleton_method(:close) do
|
|
345
|
+
original_close.call
|
|
346
|
+
ensure
|
|
347
|
+
# Finish span even if stream was closed early
|
|
348
|
+
finish_braintrust_span.call
|
|
400
349
|
end
|
|
401
350
|
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
end
|
|
406
|
-
|
|
407
|
-
# Build final content array from aggregated blocks
|
|
408
|
-
content_builders.each do |index, text|
|
|
409
|
-
block = content_blocks[index]
|
|
410
|
-
next unless block
|
|
411
|
-
|
|
412
|
-
block_type = block[:type] || block["type"]
|
|
413
|
-
case block_type
|
|
414
|
-
when "text"
|
|
415
|
-
block[:text] = text
|
|
416
|
-
block["text"] = text
|
|
417
|
-
when "tool_use"
|
|
418
|
-
# Parse the accumulated JSON string
|
|
419
|
-
begin
|
|
420
|
-
parsed = JSON.parse(text)
|
|
421
|
-
block[:input] = parsed
|
|
422
|
-
block["input"] = parsed
|
|
423
|
-
rescue JSON::ParserError
|
|
424
|
-
block[:input] = text
|
|
425
|
-
block["input"] = text
|
|
351
|
+
# If a block was provided to stream(), call each with it immediately
|
|
352
|
+
if block
|
|
353
|
+
stream.each(&block)
|
|
426
354
|
end
|
|
427
|
-
end
|
|
428
|
-
end
|
|
429
355
|
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
result[:content] = content_blocks.keys.sort.map { |idx| content_blocks[idx] }
|
|
356
|
+
stream
|
|
357
|
+
end
|
|
433
358
|
end
|
|
434
359
|
|
|
435
|
-
|
|
360
|
+
# Prepend the wrapper to the messages resource
|
|
361
|
+
client.messages.singleton_class.prepend(wrapper)
|
|
436
362
|
end
|
|
437
363
|
end
|
|
438
364
|
end
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "opentelemetry/sdk"
|
|
4
|
+
require "json"
|
|
5
|
+
|
|
6
|
+
module Braintrust
|
|
7
|
+
module Trace
|
|
8
|
+
module AlexRudall
|
|
9
|
+
module RubyOpenAI
|
|
10
|
+
# Helper to safely set a JSON attribute on a span
|
|
11
|
+
# Only sets the attribute if obj is present
|
|
12
|
+
# @param span [OpenTelemetry::Trace::Span] the span to set attribute on
|
|
13
|
+
# @param attr_name [String] the attribute name (e.g., "braintrust.output_json")
|
|
14
|
+
# @param obj [Object] the object to serialize to JSON
|
|
15
|
+
# @return [void]
|
|
16
|
+
def self.set_json_attr(span, attr_name, obj)
|
|
17
|
+
return unless obj
|
|
18
|
+
span.set_attribute(attr_name, JSON.generate(obj))
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Parse usage tokens from OpenAI API response, handling nested token_details
|
|
22
|
+
# Maps OpenAI field names to Braintrust standard names:
|
|
23
|
+
# - prompt_tokens → prompt_tokens
|
|
24
|
+
# - completion_tokens → completion_tokens
|
|
25
|
+
# - total_tokens → tokens
|
|
26
|
+
#
|
|
27
|
+
# @param usage [Hash] usage hash from OpenAI response
|
|
28
|
+
# @return [Hash<String, Integer>] metrics hash with normalized names
|
|
29
|
+
def self.parse_usage_tokens(usage)
|
|
30
|
+
metrics = {}
|
|
31
|
+
return metrics unless usage
|
|
32
|
+
|
|
33
|
+
# Basic token counts
|
|
34
|
+
metrics["prompt_tokens"] = usage["prompt_tokens"].to_i if usage["prompt_tokens"]
|
|
35
|
+
metrics["completion_tokens"] = usage["completion_tokens"].to_i if usage["completion_tokens"]
|
|
36
|
+
metrics["total_tokens"] = usage["total_tokens"].to_i if usage["total_tokens"]
|
|
37
|
+
|
|
38
|
+
# Rename total_tokens to tokens for consistency
|
|
39
|
+
metrics["tokens"] = metrics.delete("total_tokens") if metrics["total_tokens"]
|
|
40
|
+
|
|
41
|
+
metrics
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Wrap an OpenAI::Client (ruby-openai gem) to automatically create spans
|
|
45
|
+
# Supports both synchronous and streaming requests
|
|
46
|
+
# @param client [OpenAI::Client] the OpenAI client to wrap
|
|
47
|
+
# @param tracer_provider [OpenTelemetry::SDK::Trace::TracerProvider] the tracer provider (defaults to global)
|
|
48
|
+
def self.wrap(client, tracer_provider: nil)
|
|
49
|
+
tracer_provider ||= ::OpenTelemetry.tracer_provider
|
|
50
|
+
|
|
51
|
+
# Wrap chat completions
|
|
52
|
+
wrap_chat(client, tracer_provider)
|
|
53
|
+
|
|
54
|
+
client
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Wrap chat API
|
|
58
|
+
# @param client [OpenAI::Client] the OpenAI client
|
|
59
|
+
# @param tracer_provider [OpenTelemetry::SDK::Trace::TracerProvider] the tracer provider
|
|
60
|
+
def self.wrap_chat(client, tracer_provider)
|
|
61
|
+
# Create a wrapper module that intercepts the chat method
|
|
62
|
+
wrapper = Module.new do
|
|
63
|
+
define_method(:chat) do |parameters:|
|
|
64
|
+
tracer = tracer_provider.tracer("braintrust")
|
|
65
|
+
|
|
66
|
+
tracer.in_span("openai.chat.completions.create") do |span|
|
|
67
|
+
# Initialize metadata hash
|
|
68
|
+
metadata = {
|
|
69
|
+
"provider" => "openai",
|
|
70
|
+
"endpoint" => "/v1/chat/completions"
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
# Capture request metadata fields
|
|
74
|
+
metadata_fields = %w[
|
|
75
|
+
model frequency_penalty logit_bias logprobs max_tokens n
|
|
76
|
+
presence_penalty response_format seed service_tier stop
|
|
77
|
+
stream stream_options temperature top_p top_logprobs
|
|
78
|
+
tools tool_choice parallel_tool_calls user functions function_call
|
|
79
|
+
]
|
|
80
|
+
|
|
81
|
+
metadata_fields.each do |field|
|
|
82
|
+
field_sym = field.to_sym
|
|
83
|
+
if parameters.key?(field_sym)
|
|
84
|
+
# Special handling for stream parameter (it's a Proc)
|
|
85
|
+
metadata[field] = if field == "stream"
|
|
86
|
+
true # Just mark as streaming
|
|
87
|
+
else
|
|
88
|
+
parameters[field_sym]
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Set input messages as JSON
|
|
94
|
+
if parameters[:messages]
|
|
95
|
+
span.set_attribute("braintrust.input_json", JSON.generate(parameters[:messages]))
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
begin
|
|
99
|
+
# Call the original method
|
|
100
|
+
response = super(parameters: parameters)
|
|
101
|
+
|
|
102
|
+
# Set output (choices) as JSON
|
|
103
|
+
if response && response["choices"]&.any?
|
|
104
|
+
span.set_attribute("braintrust.output_json", JSON.generate(response["choices"]))
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Set metrics (token usage)
|
|
108
|
+
if response && response["usage"]
|
|
109
|
+
metrics = Braintrust::Trace::AlexRudall::RubyOpenAI.parse_usage_tokens(response["usage"])
|
|
110
|
+
span.set_attribute("braintrust.metrics", JSON.generate(metrics)) unless metrics.empty?
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
# Add response metadata fields
|
|
114
|
+
if response
|
|
115
|
+
metadata["id"] = response["id"] if response["id"]
|
|
116
|
+
metadata["created"] = response["created"] if response["created"]
|
|
117
|
+
metadata["system_fingerprint"] = response["system_fingerprint"] if response["system_fingerprint"]
|
|
118
|
+
metadata["service_tier"] = response["service_tier"] if response["service_tier"]
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
# Set metadata ONCE at the end with complete hash
|
|
122
|
+
span.set_attribute("braintrust.metadata", JSON.generate(metadata))
|
|
123
|
+
|
|
124
|
+
response
|
|
125
|
+
rescue => e
|
|
126
|
+
# Record exception in span
|
|
127
|
+
span.record_exception(e)
|
|
128
|
+
span.status = OpenTelemetry::Trace::Status.error("Exception: #{e.class} - #{e.message}")
|
|
129
|
+
raise
|
|
130
|
+
end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Prepend the wrapper to the client's singleton class
|
|
136
|
+
client.singleton_class.prepend(wrapper)
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
end
|
|
141
|
+
end
|
|
@@ -124,7 +124,7 @@ module Braintrust
|
|
|
124
124
|
choice_data[index] ||= {
|
|
125
125
|
index: index,
|
|
126
126
|
role: nil,
|
|
127
|
-
content: "",
|
|
127
|
+
content: +"",
|
|
128
128
|
tool_calls: [],
|
|
129
129
|
finish_reason: nil
|
|
130
130
|
}
|
|
@@ -136,7 +136,7 @@ module Braintrust
|
|
|
136
136
|
|
|
137
137
|
# Aggregate content
|
|
138
138
|
if delta[:content]
|
|
139
|
-
choice_data[index][:content]
|
|
139
|
+
choice_data[index][:content] << delta[:content]
|
|
140
140
|
end
|
|
141
141
|
|
|
142
142
|
# Aggregate tool_calls (similar to Go SDK logic)
|
|
@@ -149,15 +149,15 @@ module Braintrust
|
|
|
149
149
|
id: tool_call_delta[:id],
|
|
150
150
|
type: tool_call_delta[:type],
|
|
151
151
|
function: {
|
|
152
|
-
name: tool_call_delta.dig(:function, :name) || "",
|
|
153
|
-
arguments: tool_call_delta.dig(:function, :arguments) || ""
|
|
152
|
+
name: tool_call_delta.dig(:function, :name) || +"",
|
|
153
|
+
arguments: tool_call_delta.dig(:function, :arguments) || +""
|
|
154
154
|
}
|
|
155
155
|
}
|
|
156
156
|
elsif choice_data[index][:tool_calls].any?
|
|
157
157
|
# Continuation - append arguments to last tool call
|
|
158
158
|
last_tool_call = choice_data[index][:tool_calls].last
|
|
159
159
|
if tool_call_delta.dig(:function, :arguments)
|
|
160
|
-
last_tool_call[:function][:arguments]
|
|
160
|
+
last_tool_call[:function][:arguments] << tool_call_delta[:function][:arguments]
|
|
161
161
|
end
|
|
162
162
|
end
|
|
163
163
|
end
|
|
@@ -353,6 +353,119 @@ module Braintrust
|
|
|
353
353
|
|
|
354
354
|
stream
|
|
355
355
|
end
|
|
356
|
+
|
|
357
|
+
# Wrap stream for streaming chat completions (returns ChatCompletionStream with convenience methods)
|
|
358
|
+
define_method(:stream) do |**params|
|
|
359
|
+
tracer = tracer_provider.tracer("braintrust")
|
|
360
|
+
metadata = {
|
|
361
|
+
"provider" => "openai",
|
|
362
|
+
"endpoint" => "/v1/chat/completions"
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
# Start span with proper context (will be child of current span if any)
|
|
366
|
+
span = tracer.start_span("openai.chat.completions.create")
|
|
367
|
+
|
|
368
|
+
# Capture request metadata fields
|
|
369
|
+
metadata_fields = %i[
|
|
370
|
+
model frequency_penalty logit_bias logprobs max_tokens n
|
|
371
|
+
presence_penalty response_format seed service_tier stop
|
|
372
|
+
stream stream_options temperature top_p top_logprobs
|
|
373
|
+
tools tool_choice parallel_tool_calls user functions function_call
|
|
374
|
+
]
|
|
375
|
+
|
|
376
|
+
metadata_fields.each do |field|
|
|
377
|
+
metadata[field.to_s] = params[field] if params.key?(field)
|
|
378
|
+
end
|
|
379
|
+
metadata["stream"] = true # Explicitly mark as streaming
|
|
380
|
+
|
|
381
|
+
# Set input messages as JSON
|
|
382
|
+
if params[:messages]
|
|
383
|
+
messages_array = params[:messages].map(&:to_h)
|
|
384
|
+
span.set_attribute("braintrust.input_json", JSON.generate(messages_array))
|
|
385
|
+
end
|
|
386
|
+
|
|
387
|
+
# Set initial metadata
|
|
388
|
+
span.set_attribute("braintrust.metadata", JSON.generate(metadata))
|
|
389
|
+
|
|
390
|
+
# Call the original stream method with error handling
|
|
391
|
+
begin
|
|
392
|
+
stream = super(**params)
|
|
393
|
+
rescue => e
|
|
394
|
+
# Record exception if stream creation fails
|
|
395
|
+
span.record_exception(e)
|
|
396
|
+
span.status = ::OpenTelemetry::Trace::Status.error("OpenAI API error: #{e.message}")
|
|
397
|
+
span.finish
|
|
398
|
+
raise
|
|
399
|
+
end
|
|
400
|
+
|
|
401
|
+
# Local helper for setting JSON attributes
|
|
402
|
+
set_json_attr = ->(attr_name, obj) { Braintrust::Trace::OpenAI.set_json_attr(span, attr_name, obj) }
|
|
403
|
+
|
|
404
|
+
# Helper to extract metadata from SDK's internal snapshot
|
|
405
|
+
extract_stream_metadata = lambda do
|
|
406
|
+
# Access the SDK's internal accumulated completion snapshot
|
|
407
|
+
snapshot = stream.current_completion_snapshot
|
|
408
|
+
return unless snapshot
|
|
409
|
+
|
|
410
|
+
# Set output from accumulated choices
|
|
411
|
+
if snapshot.choices&.any?
|
|
412
|
+
choices_array = snapshot.choices.map(&:to_h)
|
|
413
|
+
set_json_attr.call("braintrust.output_json", choices_array)
|
|
414
|
+
end
|
|
415
|
+
|
|
416
|
+
# Set metrics if usage is available
|
|
417
|
+
if snapshot.usage
|
|
418
|
+
metrics = Braintrust::Trace::OpenAI.parse_usage_tokens(snapshot.usage)
|
|
419
|
+
set_json_attr.call("braintrust.metrics", metrics) unless metrics.empty?
|
|
420
|
+
end
|
|
421
|
+
|
|
422
|
+
# Update metadata with response fields
|
|
423
|
+
metadata["id"] = snapshot.id if snapshot.respond_to?(:id) && snapshot.id
|
|
424
|
+
metadata["created"] = snapshot.created if snapshot.respond_to?(:created) && snapshot.created
|
|
425
|
+
metadata["model"] = snapshot.model if snapshot.respond_to?(:model) && snapshot.model
|
|
426
|
+
metadata["system_fingerprint"] = snapshot.system_fingerprint if snapshot.respond_to?(:system_fingerprint) && snapshot.system_fingerprint
|
|
427
|
+
set_json_attr.call("braintrust.metadata", metadata)
|
|
428
|
+
end
|
|
429
|
+
|
|
430
|
+
# Prevent double-finish of span
|
|
431
|
+
finish_braintrust_span = lambda do
|
|
432
|
+
return if stream.instance_variable_get(:@braintrust_span_finished)
|
|
433
|
+
stream.instance_variable_set(:@braintrust_span_finished, true)
|
|
434
|
+
extract_stream_metadata.call
|
|
435
|
+
span.finish
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
# Wrap .each() method - this is the core consumption method
|
|
439
|
+
original_each = stream.method(:each)
|
|
440
|
+
stream.define_singleton_method(:each) do |&block|
|
|
441
|
+
original_each.call(&block)
|
|
442
|
+
rescue => e
|
|
443
|
+
span.record_exception(e)
|
|
444
|
+
span.status = ::OpenTelemetry::Trace::Status.error("Streaming error: #{e.message}")
|
|
445
|
+
raise
|
|
446
|
+
ensure
|
|
447
|
+
finish_braintrust_span.call
|
|
448
|
+
end
|
|
449
|
+
|
|
450
|
+
# Wrap .text() method - returns enumerable for text deltas
|
|
451
|
+
original_text = stream.method(:text)
|
|
452
|
+
stream.define_singleton_method(:text) do
|
|
453
|
+
text_enum = original_text.call
|
|
454
|
+
# Wrap the returned enumerable's .each method
|
|
455
|
+
text_enum.define_singleton_method(:each) do |&block|
|
|
456
|
+
super(&block)
|
|
457
|
+
rescue => e
|
|
458
|
+
span.record_exception(e)
|
|
459
|
+
span.status = ::OpenTelemetry::Trace::Status.error("Streaming error: #{e.message}")
|
|
460
|
+
raise
|
|
461
|
+
ensure
|
|
462
|
+
finish_braintrust_span.call
|
|
463
|
+
end
|
|
464
|
+
text_enum
|
|
465
|
+
end
|
|
466
|
+
|
|
467
|
+
stream
|
|
468
|
+
end
|
|
356
469
|
end
|
|
357
470
|
|
|
358
471
|
# Prepend the wrapper to the completions resource
|
data/lib/braintrust/trace.rb
CHANGED
|
@@ -6,12 +6,30 @@ require_relative "trace/span_processor"
|
|
|
6
6
|
require_relative "trace/span_filter"
|
|
7
7
|
require_relative "logger"
|
|
8
8
|
|
|
9
|
-
# OpenAI
|
|
9
|
+
# OpenAI integrations - both ruby-openai and openai gems use require "openai"
|
|
10
|
+
# so we detect which one actually loaded the code and require the appropriate integration
|
|
10
11
|
begin
|
|
11
12
|
require "openai"
|
|
12
|
-
|
|
13
|
+
|
|
14
|
+
# Check which OpenAI gem's code is actually loaded by inspecting $LOADED_FEATURES
|
|
15
|
+
# (both gems can be in Gem.loaded_specs, but only one's code can be loaded)
|
|
16
|
+
openai_load_path = $LOADED_FEATURES.find { |f| f.end_with?("/openai.rb") }
|
|
17
|
+
|
|
18
|
+
if openai_load_path&.include?("ruby-openai")
|
|
19
|
+
# alexrudall/ruby-openai gem (path contains "ruby-openai-X.Y.Z")
|
|
20
|
+
require_relative "trace/contrib/github.com/alexrudall/ruby-openai/ruby-openai"
|
|
21
|
+
elsif openai_load_path&.include?("/openai-")
|
|
22
|
+
# Official openai gem (path contains "openai-X.Y.Z")
|
|
23
|
+
require_relative "trace/contrib/openai"
|
|
24
|
+
elsif Gem.loaded_specs["ruby-openai"]
|
|
25
|
+
# Fallback: ruby-openai in loaded_specs (for unusual installation paths)
|
|
26
|
+
require_relative "trace/contrib/github.com/alexrudall/ruby-openai/ruby-openai"
|
|
27
|
+
elsif Gem.loaded_specs["openai"]
|
|
28
|
+
# Fallback: official openai in loaded_specs (for unusual installation paths)
|
|
29
|
+
require_relative "trace/contrib/openai"
|
|
30
|
+
end
|
|
13
31
|
rescue LoadError
|
|
14
|
-
# OpenAI gem
|
|
32
|
+
# No OpenAI gem installed - integration will not be available
|
|
15
33
|
end
|
|
16
34
|
|
|
17
35
|
# Anthropic integration is optional - automatically loaded if anthropic gem is available
|
data/lib/braintrust/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: braintrust
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Braintrust
|
|
@@ -15,14 +15,14 @@ dependencies:
|
|
|
15
15
|
requirements:
|
|
16
16
|
- - "~>"
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
|
-
version: '1.
|
|
18
|
+
version: '1.3'
|
|
19
19
|
type: :runtime
|
|
20
20
|
prerelease: false
|
|
21
21
|
version_requirements: !ruby/object:Gem::Requirement
|
|
22
22
|
requirements:
|
|
23
23
|
- - "~>"
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
|
-
version: '1.
|
|
25
|
+
version: '1.3'
|
|
26
26
|
- !ruby/object:Gem::Dependency
|
|
27
27
|
name: opentelemetry-exporter-otlp
|
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -204,6 +204,7 @@ files:
|
|
|
204
204
|
- lib/braintrust/trace.rb
|
|
205
205
|
- lib/braintrust/trace/attachment.rb
|
|
206
206
|
- lib/braintrust/trace/contrib/anthropic.rb
|
|
207
|
+
- lib/braintrust/trace/contrib/github.com/alexrudall/ruby-openai/ruby-openai.rb
|
|
207
208
|
- lib/braintrust/trace/contrib/openai.rb
|
|
208
209
|
- lib/braintrust/trace/span_filter.rb
|
|
209
210
|
- lib/braintrust/trace/span_processor.rb
|