braintrust 0.0.3 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +53 -2
- data/lib/braintrust/config.rb +21 -4
- data/lib/braintrust/eval.rb +164 -0
- data/lib/braintrust/state.rb +14 -6
- data/lib/braintrust/trace/attachment.rb +138 -0
- data/lib/braintrust/trace/contrib/anthropic.rb +82 -156
- data/lib/braintrust/trace/contrib/github.com/alexrudall/ruby-openai/ruby-openai.rb +141 -0
- data/lib/braintrust/trace/contrib/openai.rb +118 -5
- data/lib/braintrust/trace/span_filter.rb +59 -0
- data/lib/braintrust/trace/span_processor.rb +29 -3
- data/lib/braintrust/trace.rb +60 -10
- data/lib/braintrust/version.rb +1 -1
- data/lib/braintrust.rb +8 -2
- metadata +6 -3
|
@@ -5,14 +5,16 @@ require "opentelemetry/sdk"
|
|
|
5
5
|
module Braintrust
|
|
6
6
|
module Trace
|
|
7
7
|
# Custom span processor that adds Braintrust-specific attributes to spans
|
|
8
|
+
# and optionally filters spans based on custom filter functions.
|
|
8
9
|
class SpanProcessor
|
|
9
10
|
PARENT_ATTR_KEY = "braintrust.parent"
|
|
10
11
|
ORG_ATTR_KEY = "braintrust.org"
|
|
11
12
|
APP_URL_ATTR_KEY = "braintrust.app_url"
|
|
12
13
|
|
|
13
|
-
def initialize(wrapped_processor, state)
|
|
14
|
+
def initialize(wrapped_processor, state, filters = [])
|
|
14
15
|
@wrapped = wrapped_processor
|
|
15
16
|
@state = state
|
|
17
|
+
@filters = filters || []
|
|
16
18
|
end
|
|
17
19
|
|
|
18
20
|
def on_start(span, parent_context)
|
|
@@ -33,9 +35,10 @@ module Braintrust
|
|
|
33
35
|
@wrapped.on_start(span, parent_context)
|
|
34
36
|
end
|
|
35
37
|
|
|
36
|
-
# Called when a span ends
|
|
38
|
+
# Called when a span ends - apply filters before forwarding
|
|
37
39
|
def on_finish(span)
|
|
38
|
-
|
|
40
|
+
# Only forward span if it passes filters
|
|
41
|
+
@wrapped.on_finish(span) if should_forward_span?(span)
|
|
39
42
|
end
|
|
40
43
|
|
|
41
44
|
# Shutdown the processor
|
|
@@ -73,6 +76,29 @@ module Braintrust
|
|
|
73
76
|
# Return the parent attribute from the parent span
|
|
74
77
|
parent_span.attributes&.[](PARENT_ATTR_KEY)
|
|
75
78
|
end
|
|
79
|
+
|
|
80
|
+
# Determine if a span should be forwarded to the wrapped processor
|
|
81
|
+
# based on configured filters
|
|
82
|
+
def should_forward_span?(span)
|
|
83
|
+
# Always keep root spans (spans with no parent)
|
|
84
|
+
# Check if parent_span_id is the invalid/zero span ID
|
|
85
|
+
is_root = span.parent_span_id == OpenTelemetry::Trace::INVALID_SPAN_ID
|
|
86
|
+
return true if is_root
|
|
87
|
+
|
|
88
|
+
# If no filters, keep everything
|
|
89
|
+
return true if @filters.empty?
|
|
90
|
+
|
|
91
|
+
# Apply filters in order - first non-zero result wins
|
|
92
|
+
@filters.each do |filter|
|
|
93
|
+
result = filter.call(span)
|
|
94
|
+
return true if result > 0 # Keep span
|
|
95
|
+
return false if result < 0 # Drop span
|
|
96
|
+
# result == 0: no influence, continue to next filter
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# All filters returned 0 (no influence), default to keep
|
|
100
|
+
true
|
|
101
|
+
end
|
|
76
102
|
end
|
|
77
103
|
end
|
|
78
104
|
end
|
data/lib/braintrust/trace.rb
CHANGED
|
@@ -3,14 +3,33 @@
|
|
|
3
3
|
require "opentelemetry/sdk"
|
|
4
4
|
require "opentelemetry/exporter/otlp"
|
|
5
5
|
require_relative "trace/span_processor"
|
|
6
|
+
require_relative "trace/span_filter"
|
|
6
7
|
require_relative "logger"
|
|
7
8
|
|
|
8
|
-
# OpenAI
|
|
9
|
+
# OpenAI integrations - both ruby-openai and openai gems use require "openai"
|
|
10
|
+
# so we detect which one actually loaded the code and require the appropriate integration
|
|
9
11
|
begin
|
|
10
12
|
require "openai"
|
|
11
|
-
|
|
13
|
+
|
|
14
|
+
# Check which OpenAI gem's code is actually loaded by inspecting $LOADED_FEATURES
|
|
15
|
+
# (both gems can be in Gem.loaded_specs, but only one's code can be loaded)
|
|
16
|
+
openai_load_path = $LOADED_FEATURES.find { |f| f.end_with?("/openai.rb") }
|
|
17
|
+
|
|
18
|
+
if openai_load_path&.include?("ruby-openai")
|
|
19
|
+
# alexrudall/ruby-openai gem (path contains "ruby-openai-X.Y.Z")
|
|
20
|
+
require_relative "trace/contrib/github.com/alexrudall/ruby-openai/ruby-openai"
|
|
21
|
+
elsif openai_load_path&.include?("/openai-")
|
|
22
|
+
# Official openai gem (path contains "openai-X.Y.Z")
|
|
23
|
+
require_relative "trace/contrib/openai"
|
|
24
|
+
elsif Gem.loaded_specs["ruby-openai"]
|
|
25
|
+
# Fallback: ruby-openai in loaded_specs (for unusual installation paths)
|
|
26
|
+
require_relative "trace/contrib/github.com/alexrudall/ruby-openai/ruby-openai"
|
|
27
|
+
elsif Gem.loaded_specs["openai"]
|
|
28
|
+
# Fallback: official openai in loaded_specs (for unusual installation paths)
|
|
29
|
+
require_relative "trace/contrib/openai"
|
|
30
|
+
end
|
|
12
31
|
rescue LoadError
|
|
13
|
-
# OpenAI gem
|
|
32
|
+
# No OpenAI gem installed - integration will not be available
|
|
14
33
|
end
|
|
15
34
|
|
|
16
35
|
# Anthropic integration is optional - automatically loaded if anthropic gem is available
|
|
@@ -26,8 +45,9 @@ module Braintrust
|
|
|
26
45
|
# Set up OpenTelemetry tracing with Braintrust
|
|
27
46
|
# @param state [State] Braintrust state
|
|
28
47
|
# @param tracer_provider [TracerProvider, nil] Optional tracer provider
|
|
48
|
+
# @param exporter [Exporter, nil] Optional exporter override (for testing)
|
|
29
49
|
# @return [void]
|
|
30
|
-
def self.setup(state, tracer_provider = nil)
|
|
50
|
+
def self.setup(state, tracer_provider = nil, exporter: nil)
|
|
31
51
|
if tracer_provider
|
|
32
52
|
# Use the explicitly provided tracer provider
|
|
33
53
|
# DO NOT set as global - user is managing it themselves
|
|
@@ -49,13 +69,17 @@ module Braintrust
|
|
|
49
69
|
end
|
|
50
70
|
|
|
51
71
|
# Enable Braintrust tracing (adds span processor)
|
|
52
|
-
|
|
72
|
+
config = state.config
|
|
73
|
+
enable(tracer_provider, state: state, config: config, exporter: exporter)
|
|
53
74
|
end
|
|
54
75
|
|
|
55
|
-
def self.enable(tracer_provider, state: nil, exporter: nil)
|
|
76
|
+
def self.enable(tracer_provider, state: nil, exporter: nil, config: nil)
|
|
56
77
|
state ||= Braintrust.current_state
|
|
57
78
|
raise Error, "No state available" unless state
|
|
58
79
|
|
|
80
|
+
# Get config from state if available
|
|
81
|
+
config ||= state.respond_to?(:config) ? state.config : nil
|
|
82
|
+
|
|
59
83
|
# Create OTLP HTTP exporter unless override provided
|
|
60
84
|
exporter ||= OpenTelemetry::Exporter::OTLP::Exporter.new(
|
|
61
85
|
endpoint: "#{state.api_url}/otel/v1/traces",
|
|
@@ -64,11 +88,18 @@ module Braintrust
|
|
|
64
88
|
}
|
|
65
89
|
)
|
|
66
90
|
|
|
67
|
-
#
|
|
68
|
-
|
|
91
|
+
# Use SimpleSpanProcessor for InMemorySpanExporter (testing), BatchSpanProcessor for production
|
|
92
|
+
span_processor = if exporter.is_a?(OpenTelemetry::SDK::Trace::Export::InMemorySpanExporter)
|
|
93
|
+
OpenTelemetry::SDK::Trace::Export::SimpleSpanProcessor.new(exporter)
|
|
94
|
+
else
|
|
95
|
+
OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(exporter)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Build filters array from config
|
|
99
|
+
filters = build_filters(config)
|
|
69
100
|
|
|
70
|
-
# Wrap
|
|
71
|
-
processor = SpanProcessor.new(
|
|
101
|
+
# Wrap span processor in our custom span processor to add Braintrust attributes and filters
|
|
102
|
+
processor = SpanProcessor.new(span_processor, state, filters)
|
|
72
103
|
|
|
73
104
|
# Register with tracer provider
|
|
74
105
|
tracer_provider.add_span_processor(processor)
|
|
@@ -83,6 +114,25 @@ module Braintrust
|
|
|
83
114
|
self
|
|
84
115
|
end
|
|
85
116
|
|
|
117
|
+
# Build filters array from config
|
|
118
|
+
# @param config [Config, nil] Configuration object
|
|
119
|
+
# @return [Array<Proc>] Array of filter functions
|
|
120
|
+
def self.build_filters(config)
|
|
121
|
+
filters = []
|
|
122
|
+
|
|
123
|
+
# Add custom filters first (they have priority)
|
|
124
|
+
if config&.span_filter_funcs&.any?
|
|
125
|
+
filters.concat(config.span_filter_funcs)
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
# Add AI filter if enabled
|
|
129
|
+
if config&.filter_ai_spans
|
|
130
|
+
filters << SpanFilter.method(:ai_filter)
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
filters
|
|
134
|
+
end
|
|
135
|
+
|
|
86
136
|
# Generate a permalink URL for a span to view in the Braintrust UI
|
|
87
137
|
# Returns an empty string if the permalink cannot be generated
|
|
88
138
|
# @param span [OpenTelemetry::Trace::Span] The span to generate a permalink for
|
data/lib/braintrust/version.rb
CHANGED
data/lib/braintrust.rb
CHANGED
|
@@ -37,8 +37,11 @@ module Braintrust
|
|
|
37
37
|
# @param blocking_login [Boolean] Whether to block and login synchronously (default: false - async background login)
|
|
38
38
|
# @param enable_tracing [Boolean] Whether to enable OpenTelemetry tracing (default: true)
|
|
39
39
|
# @param tracer_provider [TracerProvider, nil] Optional tracer provider to use instead of creating one
|
|
40
|
+
# @param filter_ai_spans [Boolean, nil] Enable AI span filtering (overrides BRAINTRUST_OTEL_FILTER_AI_SPANS env var)
|
|
41
|
+
# @param span_filter_funcs [Array<Proc>, nil] Custom span filter functions
|
|
42
|
+
# @param exporter [Exporter, nil] Optional exporter override (for testing)
|
|
40
43
|
# @return [State] the created state
|
|
41
|
-
def self.init(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil, set_global: true, blocking_login: false, enable_tracing: true, tracer_provider: nil)
|
|
44
|
+
def self.init(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil, set_global: true, blocking_login: false, enable_tracing: true, tracer_provider: nil, filter_ai_spans: nil, span_filter_funcs: nil, exporter: nil)
|
|
42
45
|
state = State.from_env(
|
|
43
46
|
api_key: api_key,
|
|
44
47
|
org_name: org_name,
|
|
@@ -47,7 +50,10 @@ module Braintrust
|
|
|
47
50
|
api_url: api_url,
|
|
48
51
|
blocking_login: blocking_login,
|
|
49
52
|
enable_tracing: enable_tracing,
|
|
50
|
-
tracer_provider: tracer_provider
|
|
53
|
+
tracer_provider: tracer_provider,
|
|
54
|
+
filter_ai_spans: filter_ai_spans,
|
|
55
|
+
span_filter_funcs: span_filter_funcs,
|
|
56
|
+
exporter: exporter
|
|
51
57
|
)
|
|
52
58
|
|
|
53
59
|
State.global = state if set_global
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: braintrust
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Braintrust
|
|
@@ -15,14 +15,14 @@ dependencies:
|
|
|
15
15
|
requirements:
|
|
16
16
|
- - "~>"
|
|
17
17
|
- !ruby/object:Gem::Version
|
|
18
|
-
version: '1.
|
|
18
|
+
version: '1.3'
|
|
19
19
|
type: :runtime
|
|
20
20
|
prerelease: false
|
|
21
21
|
version_requirements: !ruby/object:Gem::Requirement
|
|
22
22
|
requirements:
|
|
23
23
|
- - "~>"
|
|
24
24
|
- !ruby/object:Gem::Version
|
|
25
|
-
version: '1.
|
|
25
|
+
version: '1.3'
|
|
26
26
|
- !ruby/object:Gem::Dependency
|
|
27
27
|
name: opentelemetry-exporter-otlp
|
|
28
28
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -202,8 +202,11 @@ files:
|
|
|
202
202
|
- lib/braintrust/logger.rb
|
|
203
203
|
- lib/braintrust/state.rb
|
|
204
204
|
- lib/braintrust/trace.rb
|
|
205
|
+
- lib/braintrust/trace/attachment.rb
|
|
205
206
|
- lib/braintrust/trace/contrib/anthropic.rb
|
|
207
|
+
- lib/braintrust/trace/contrib/github.com/alexrudall/ruby-openai/ruby-openai.rb
|
|
206
208
|
- lib/braintrust/trace/contrib/openai.rb
|
|
209
|
+
- lib/braintrust/trace/span_filter.rb
|
|
207
210
|
- lib/braintrust/trace/span_processor.rb
|
|
208
211
|
- lib/braintrust/version.rb
|
|
209
212
|
homepage: https://github.com/braintrustdata/braintrust-sdk-ruby
|