braintrust 0.0.12 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +213 -180
- data/exe/braintrust +143 -0
- data/lib/braintrust/contrib/anthropic/deprecated.rb +24 -0
- data/lib/braintrust/contrib/anthropic/instrumentation/common.rb +53 -0
- data/lib/braintrust/contrib/anthropic/instrumentation/messages.rb +232 -0
- data/lib/braintrust/contrib/anthropic/integration.rb +53 -0
- data/lib/braintrust/contrib/anthropic/patcher.rb +62 -0
- data/lib/braintrust/contrib/context.rb +56 -0
- data/lib/braintrust/contrib/integration.rb +160 -0
- data/lib/braintrust/contrib/openai/deprecated.rb +22 -0
- data/lib/braintrust/contrib/openai/instrumentation/chat.rb +298 -0
- data/lib/braintrust/contrib/openai/instrumentation/common.rb +134 -0
- data/lib/braintrust/contrib/openai/instrumentation/responses.rb +187 -0
- data/lib/braintrust/contrib/openai/integration.rb +58 -0
- data/lib/braintrust/contrib/openai/patcher.rb +130 -0
- data/lib/braintrust/contrib/patcher.rb +76 -0
- data/lib/braintrust/contrib/rails/railtie.rb +16 -0
- data/lib/braintrust/contrib/registry.rb +107 -0
- data/lib/braintrust/contrib/ruby_llm/deprecated.rb +45 -0
- data/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb +464 -0
- data/lib/braintrust/contrib/ruby_llm/instrumentation/common.rb +58 -0
- data/lib/braintrust/contrib/ruby_llm/integration.rb +54 -0
- data/lib/braintrust/contrib/ruby_llm/patcher.rb +44 -0
- data/lib/braintrust/contrib/ruby_openai/deprecated.rb +24 -0
- data/lib/braintrust/contrib/ruby_openai/instrumentation/chat.rb +149 -0
- data/lib/braintrust/contrib/ruby_openai/instrumentation/common.rb +138 -0
- data/lib/braintrust/contrib/ruby_openai/instrumentation/responses.rb +146 -0
- data/lib/braintrust/contrib/ruby_openai/integration.rb +58 -0
- data/lib/braintrust/contrib/ruby_openai/patcher.rb +85 -0
- data/lib/braintrust/contrib/setup.rb +168 -0
- data/lib/braintrust/contrib/support/openai.rb +72 -0
- data/lib/braintrust/contrib/support/otel.rb +23 -0
- data/lib/braintrust/contrib.rb +205 -0
- data/lib/braintrust/internal/env.rb +33 -0
- data/lib/braintrust/internal/time.rb +44 -0
- data/lib/braintrust/setup.rb +50 -0
- data/lib/braintrust/state.rb +5 -0
- data/lib/braintrust/trace.rb +0 -51
- data/lib/braintrust/version.rb +1 -1
- data/lib/braintrust.rb +10 -1
- metadata +38 -7
- data/lib/braintrust/trace/contrib/anthropic.rb +0 -316
- data/lib/braintrust/trace/contrib/github.com/alexrudall/ruby-openai/ruby-openai.rb +0 -377
- data/lib/braintrust/trace/contrib/github.com/crmne/ruby_llm.rb +0 -631
- data/lib/braintrust/trace/contrib/openai.rb +0 -611
- data/lib/braintrust/trace/tokens.rb +0 -109
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "../internal/env"
|
|
4
|
+
|
|
5
|
+
module Braintrust
|
|
6
|
+
module Contrib
|
|
7
|
+
# Automatic instrumentation setup for LLM libraries.
|
|
8
|
+
#
|
|
9
|
+
# Intercepts `require` calls to detect when LLM libraries are loaded, then patches
|
|
10
|
+
# them automatically. The main challenge is doing this safely with zeitwerk (Rails'
|
|
11
|
+
# autoloader) which also hooks into require.
|
|
12
|
+
#
|
|
13
|
+
# ## The Zeitwerk Problem
|
|
14
|
+
#
|
|
15
|
+
# Zeitwerk uses `alias_method :zeitwerk_original_require, :require`. If we prepend
|
|
16
|
+
# to Kernel before zeitwerk loads, zeitwerk captures our method as its "original",
|
|
17
|
+
# creating an infinite loop.
|
|
18
|
+
#
|
|
19
|
+
# ## Solution: Two-Phase Hook
|
|
20
|
+
#
|
|
21
|
+
# ┌─────────────────────────────────────────────────────────────────────────┐
|
|
22
|
+
# │ Setup.run! called │
|
|
23
|
+
# └─────────────────────────────────────────────────────────────────────────┘
|
|
24
|
+
# │
|
|
25
|
+
# ┌─────────────────────────┼─────────────────────────┐
|
|
26
|
+
# ▼ ▼ ▼
|
|
27
|
+
# ┌─────────────────┐ ┌─────────────────────┐ ┌─────────────────────┐
|
|
28
|
+
# │ Rails loaded? │ │ Zeitwerk loaded? │ │ Neither loaded yet │
|
|
29
|
+
# │ (Rails::Railtie)│ │ │ │ │
|
|
30
|
+
# └────────┬────────┘ └──────────┬──────────┘ └──────────┬──────────┘
|
|
31
|
+
# │ │ │
|
|
32
|
+
# ▼ ▼ ▼
|
|
33
|
+
# ┌─────────────────┐ ┌─────────────────────┐ ┌─────────────────────┐
|
|
34
|
+
# │ Install Railtie │ │ Install prepend │ │ Install watcher │
|
|
35
|
+
# │ (after_init) │ │ hook directly │ │ hook (alias_method) │
|
|
36
|
+
# └─────────────────┘ └─────────────────────┘ └──────────┬──────────┘
|
|
37
|
+
# │ │ │
|
|
38
|
+
# │ │ ┌──────────┴──────────┐
|
|
39
|
+
# │ │ ▼ ▼
|
|
40
|
+
# │ │ ┌──────────────┐ ┌──────────────────┐
|
|
41
|
+
# │ │ │ Rails loads? │ │ Zeitwerk loads? │
|
|
42
|
+
# │ │ │ → Railtie │ │ → Prepend hook │
|
|
43
|
+
# │ │ └──────────────┘ └──────────────────┘
|
|
44
|
+
# │ │ │ │
|
|
45
|
+
# ▼ ▼ ▼ ▼
|
|
46
|
+
# ┌─────────────────────────────────────────────────────────────────────────┐
|
|
47
|
+
# │ LLM library loads → patch! │
|
|
48
|
+
# └─────────────────────────────────────────────────────────────────────────┘
|
|
49
|
+
#
|
|
50
|
+
# The watcher hook uses alias_method which zeitwerk captures harmlessly (alias
|
|
51
|
+
# chains work correctly). Once zeitwerk/Rails loads, we upgrade to the better
|
|
52
|
+
# approach: prepend (takes precedence, `super` chains through zeitwerk) or
|
|
53
|
+
# Railtie (patches after all gems loaded via after_initialize).
|
|
54
|
+
#
|
|
55
|
+
module Setup
|
|
56
|
+
REENTRANCY_KEY = :braintrust_in_require_hook
|
|
57
|
+
|
|
58
|
+
class << self
|
|
59
|
+
# Main entry point. Call once per process.
|
|
60
|
+
def run!
|
|
61
|
+
unless Internal::Env.auto_instrument
|
|
62
|
+
Braintrust::Log.debug("Contrib::Setup: auto-instrumentation disabled via environment")
|
|
63
|
+
return
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
@registry = Contrib::Registry.instance
|
|
67
|
+
@only = Internal::Env.instrument_only
|
|
68
|
+
@except = Internal::Env.instrument_except
|
|
69
|
+
|
|
70
|
+
if defined?(::Rails::Railtie)
|
|
71
|
+
Braintrust::Log.debug("Contrib::Setup: using Rails railtie hook")
|
|
72
|
+
install_railtie!
|
|
73
|
+
elsif defined?(::Zeitwerk)
|
|
74
|
+
Braintrust::Log.debug("Contrib::Setup: using require hook (zeitwerk detected)")
|
|
75
|
+
install_require_hook!
|
|
76
|
+
else
|
|
77
|
+
Braintrust::Log.debug("Contrib::Setup: using watcher hook")
|
|
78
|
+
install_watcher_hook!
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# Called after each require to check if we should patch anything.
|
|
83
|
+
def on_require(path)
|
|
84
|
+
return unless @registry
|
|
85
|
+
|
|
86
|
+
@registry.integrations_for_require_path(path).each do |integration|
|
|
87
|
+
next unless integration.available? && integration.compatible?
|
|
88
|
+
next if @only && !@only.include?(integration.integration_name)
|
|
89
|
+
next if @except&.include?(integration.integration_name)
|
|
90
|
+
|
|
91
|
+
Braintrust::Log.debug("Contrib::Setup: patching #{integration.integration_name}")
|
|
92
|
+
integration.patch!
|
|
93
|
+
end
|
|
94
|
+
rescue => e
|
|
95
|
+
Braintrust::Log.error("Auto-instrument failed: #{e.message}")
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
# Execute block with reentrancy protection (prevents infinite loops).
|
|
99
|
+
def with_reentrancy_guard
|
|
100
|
+
return if Thread.current[REENTRANCY_KEY]
|
|
101
|
+
Thread.current[REENTRANCY_KEY] = true
|
|
102
|
+
yield
|
|
103
|
+
rescue => e
|
|
104
|
+
Braintrust::Log.error("Auto-instrument failed: #{e.message}")
|
|
105
|
+
ensure
|
|
106
|
+
Thread.current[REENTRANCY_KEY] = false
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def railtie_installed? = @railtie_installed
|
|
110
|
+
def require_hook_installed? = @require_hook_installed
|
|
111
|
+
|
|
112
|
+
def install_railtie!
|
|
113
|
+
return if @railtie_installed
|
|
114
|
+
@railtie_installed = true
|
|
115
|
+
require_relative "rails/railtie"
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def install_require_hook!
|
|
119
|
+
return if @require_hook_installed
|
|
120
|
+
@require_hook_installed = true
|
|
121
|
+
Kernel.prepend(RequireHook)
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
private
|
|
125
|
+
|
|
126
|
+
def install_watcher_hook!
|
|
127
|
+
return if Kernel.private_method_defined?(:braintrust_watcher_require)
|
|
128
|
+
|
|
129
|
+
Kernel.module_eval do
|
|
130
|
+
alias_method :braintrust_watcher_require, :require
|
|
131
|
+
|
|
132
|
+
define_method(:require) do |path|
|
|
133
|
+
result = braintrust_watcher_require(path)
|
|
134
|
+
|
|
135
|
+
# Detect Rails/zeitwerk loading and upgrade hook strategy.
|
|
136
|
+
# IMPORTANT: Only check when the gem itself finishes loading (path match),
|
|
137
|
+
# not on every require where the constant happens to be defined.
|
|
138
|
+
# Installing too early (during gem init) breaks the alias_method chain.
|
|
139
|
+
if (path == "rails" || path.include?("railties")) &&
|
|
140
|
+
defined?(::Rails::Railtie) && !Braintrust::Contrib::Setup.railtie_installed?
|
|
141
|
+
Braintrust::Contrib::Setup.install_railtie!
|
|
142
|
+
elsif (path == "zeitwerk" || path.end_with?("/zeitwerk.rb")) &&
|
|
143
|
+
defined?(::Zeitwerk) && !Braintrust::Contrib::Setup.require_hook_installed?
|
|
144
|
+
Braintrust::Contrib::Setup.install_require_hook!
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
Braintrust::Contrib::Setup.with_reentrancy_guard do
|
|
148
|
+
Braintrust::Contrib::Setup.on_require(path)
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
result
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
# Prepend module for require interception.
|
|
159
|
+
# Only installed AFTER zeitwerk loads to avoid alias_method loop.
|
|
160
|
+
module RequireHook
|
|
161
|
+
def require(path)
|
|
162
|
+
result = super
|
|
163
|
+
Setup.with_reentrancy_guard { Setup.on_require(path) }
|
|
164
|
+
result
|
|
165
|
+
end
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Braintrust
|
|
4
|
+
module Contrib
|
|
5
|
+
module Support
|
|
6
|
+
# OpenAI API-specific utilities shared across OpenAI integrations.
|
|
7
|
+
# These work with normalized data structures (hashes), not library-specific objects.
|
|
8
|
+
module OpenAI
|
|
9
|
+
# Parse OpenAI usage tokens into normalized Braintrust metrics.
|
|
10
|
+
# Handles standard fields and *_tokens_details nested objects.
|
|
11
|
+
# Works with both Hash objects and SDK response objects (via to_h).
|
|
12
|
+
# @param usage [Hash, Object] usage object from OpenAI response
|
|
13
|
+
# @return [Hash<String, Integer>] normalized metrics
|
|
14
|
+
def self.parse_usage_tokens(usage)
|
|
15
|
+
metrics = {}
|
|
16
|
+
return metrics unless usage
|
|
17
|
+
|
|
18
|
+
usage_hash = usage.respond_to?(:to_h) ? usage.to_h : usage
|
|
19
|
+
return metrics unless usage_hash.is_a?(Hash)
|
|
20
|
+
|
|
21
|
+
# Field mappings: OpenAI → Braintrust
|
|
22
|
+
# Supports both Chat Completions API (prompt_tokens, completion_tokens)
|
|
23
|
+
# and Responses API (input_tokens, output_tokens)
|
|
24
|
+
field_map = {
|
|
25
|
+
"prompt_tokens" => "prompt_tokens",
|
|
26
|
+
"completion_tokens" => "completion_tokens",
|
|
27
|
+
"total_tokens" => "tokens",
|
|
28
|
+
# Responses API uses different field names
|
|
29
|
+
"input_tokens" => "prompt_tokens",
|
|
30
|
+
"output_tokens" => "completion_tokens"
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
# Prefix mappings for *_tokens_details
|
|
34
|
+
prefix_map = {
|
|
35
|
+
"prompt" => "prompt",
|
|
36
|
+
"completion" => "completion",
|
|
37
|
+
# Responses API uses input/output prefixes
|
|
38
|
+
"input" => "prompt",
|
|
39
|
+
"output" => "completion"
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
usage_hash.each do |key, value|
|
|
43
|
+
key_str = key.to_s
|
|
44
|
+
|
|
45
|
+
if value.is_a?(Numeric)
|
|
46
|
+
target = field_map[key_str]
|
|
47
|
+
metrics[target] = value.to_i if target
|
|
48
|
+
elsif key_str.end_with?("_tokens_details")
|
|
49
|
+
# Convert to hash if it's an object (OpenAI SDK returns objects)
|
|
50
|
+
details_hash = value.respond_to?(:to_h) ? value.to_h : value
|
|
51
|
+
next unless details_hash.is_a?(Hash)
|
|
52
|
+
|
|
53
|
+
raw_prefix = key_str.sub(/_tokens_details$/, "")
|
|
54
|
+
prefix = prefix_map[raw_prefix] || raw_prefix
|
|
55
|
+
details_hash.each do |detail_key, detail_value|
|
|
56
|
+
next unless detail_value.is_a?(Numeric)
|
|
57
|
+
metrics["#{prefix}_#{detail_key}"] = detail_value.to_i
|
|
58
|
+
end
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Calculate total if missing
|
|
63
|
+
if !metrics.key?("tokens") && metrics.key?("prompt_tokens") && metrics.key?("completion_tokens")
|
|
64
|
+
metrics["tokens"] = metrics["prompt_tokens"] + metrics["completion_tokens"]
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
metrics
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Braintrust
|
|
6
|
+
module Contrib
|
|
7
|
+
module Support
|
|
8
|
+
# OpenTelemetry utilities shared across all integrations.
|
|
9
|
+
module OTel
|
|
10
|
+
# Helper to safely set a JSON attribute on a span
|
|
11
|
+
# Only sets the attribute if obj is present
|
|
12
|
+
# @param span [OpenTelemetry::Trace::Span] the span to set attribute on
|
|
13
|
+
# @param attr_name [String] the attribute name (e.g., "braintrust.output_json")
|
|
14
|
+
# @param obj [Object] the object to serialize to JSON
|
|
15
|
+
# @return [void]
|
|
16
|
+
def self.set_json_attr(span, attr_name, obj)
|
|
17
|
+
return unless obj
|
|
18
|
+
span.set_attribute(attr_name, JSON.generate(obj))
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "contrib/registry"
|
|
4
|
+
require_relative "contrib/integration"
|
|
5
|
+
require_relative "contrib/patcher"
|
|
6
|
+
require_relative "contrib/context"
|
|
7
|
+
|
|
8
|
+
module Braintrust
|
|
9
|
+
# Auto-instrument available integrations.
|
|
10
|
+
#
|
|
11
|
+
# Discovers which integrations have their target libraries loaded
|
|
12
|
+
# and instruments them automatically. This is the primary public API
|
|
13
|
+
# for enabling auto-instrumentation.
|
|
14
|
+
#
|
|
15
|
+
# @param config [nil, Boolean, Hash] Auto-instrumentation configuration:
|
|
16
|
+
# - nil: use BRAINTRUST_AUTO_INSTRUMENT env var (defaults to enabled)
|
|
17
|
+
# - true: explicitly enable auto-instrumentation
|
|
18
|
+
# - false: explicitly disable auto-instrumentation
|
|
19
|
+
# - Hash with :only or :except keys for filtering integrations
|
|
20
|
+
#
|
|
21
|
+
# Environment variables:
|
|
22
|
+
# - BRAINTRUST_AUTO_INSTRUMENT: set to "false" to disable (default: enabled)
|
|
23
|
+
# - BRAINTRUST_INSTRUMENT_ONLY: comma-separated list of integrations to include
|
|
24
|
+
# - BRAINTRUST_INSTRUMENT_EXCEPT: comma-separated list of integrations to exclude
|
|
25
|
+
#
|
|
26
|
+
# @return [Array<Symbol>, nil] names of instrumented integrations, or nil if disabled
|
|
27
|
+
#
|
|
28
|
+
# @example Enable with defaults
|
|
29
|
+
# Braintrust.auto_instrument!
|
|
30
|
+
#
|
|
31
|
+
# @example Explicitly enable
|
|
32
|
+
# Braintrust.auto_instrument!(true)
|
|
33
|
+
#
|
|
34
|
+
# @example Disable
|
|
35
|
+
# Braintrust.auto_instrument!(false)
|
|
36
|
+
#
|
|
37
|
+
# @example Only specific integrations
|
|
38
|
+
# Braintrust.auto_instrument!(only: [:openai, :anthropic])
|
|
39
|
+
#
|
|
40
|
+
# @example Exclude specific integrations
|
|
41
|
+
# Braintrust.auto_instrument!(except: [:ruby_llm])
|
|
42
|
+
def self.auto_instrument!(config = nil)
|
|
43
|
+
should_instrument = case config
|
|
44
|
+
when nil
|
|
45
|
+
Internal::Env.auto_instrument
|
|
46
|
+
when false
|
|
47
|
+
false
|
|
48
|
+
when true, Hash
|
|
49
|
+
true
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
return unless should_instrument
|
|
53
|
+
|
|
54
|
+
only = Internal::Env.instrument_only
|
|
55
|
+
except = Internal::Env.instrument_except
|
|
56
|
+
|
|
57
|
+
if config.is_a?(Hash)
|
|
58
|
+
only = config[:only] || only
|
|
59
|
+
except = config[:except] || except
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
Contrib.auto_instrument!(only: only, except: except)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# Instrument a registered integration by name.
|
|
66
|
+
# This is the main entry point for activating integrations.
|
|
67
|
+
#
|
|
68
|
+
# @param name [Symbol] The integration name (e.g., :openai, :anthropic)
|
|
69
|
+
# @param options [Hash] Optional configuration
|
|
70
|
+
# @option options [Object] :target Optional target instance to instrument specifically
|
|
71
|
+
# @option options [OpenTelemetry::SDK::Trace::TracerProvider] :tracer_provider Optional tracer provider
|
|
72
|
+
# @return [void]
|
|
73
|
+
#
|
|
74
|
+
# @example Instrument all OpenAI clients
|
|
75
|
+
# Braintrust.instrument!(:openai)
|
|
76
|
+
#
|
|
77
|
+
# @example Instrument specific OpenAI client instance
|
|
78
|
+
# client = OpenAI::Client.new
|
|
79
|
+
# Braintrust.instrument!(:openai, target: client, tracer_provider: my_provider)
|
|
80
|
+
def self.instrument!(name, **options)
|
|
81
|
+
Contrib.instrument!(name, **options)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Contrib framework for auto-instrumentation integrations.
|
|
85
|
+
# Provides a consistent interface for all integrations and enables
|
|
86
|
+
# reliable auto-instrumentation in later milestones.
|
|
87
|
+
module Contrib
|
|
88
|
+
class << self
|
|
89
|
+
# Get the global registry instance.
|
|
90
|
+
# @return [Registry]
|
|
91
|
+
def registry
|
|
92
|
+
Registry.instance
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
# Initialize the contrib framework with optional configuration.
|
|
96
|
+
# @param tracer_provider [OpenTelemetry::SDK::Trace::TracerProvider, nil] Optional tracer provider
|
|
97
|
+
# @return [void]
|
|
98
|
+
def init(tracer_provider: nil)
|
|
99
|
+
@default_tracer_provider = tracer_provider
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Auto-instrument available integrations.
|
|
103
|
+
# Discovers which integrations have their target libraries loaded
|
|
104
|
+
# and instruments them automatically.
|
|
105
|
+
#
|
|
106
|
+
# @param only [Array<Symbol>] whitelist - only instrument these
|
|
107
|
+
# @param except [Array<Symbol>] blacklist - skip these
|
|
108
|
+
# @return [Array<Symbol>] names of integrations that were instrumented
|
|
109
|
+
#
|
|
110
|
+
# @example Instrument all available
|
|
111
|
+
# Braintrust::Contrib.auto_instrument!
|
|
112
|
+
#
|
|
113
|
+
# @example Only specific integrations
|
|
114
|
+
# Braintrust::Contrib.auto_instrument!(only: [:openai, :anthropic])
|
|
115
|
+
#
|
|
116
|
+
# @example Exclude specific integrations
|
|
117
|
+
# Braintrust::Contrib.auto_instrument!(except: [:ruby_llm])
|
|
118
|
+
def auto_instrument!(only: nil, except: nil)
|
|
119
|
+
if only || except
|
|
120
|
+
Braintrust::Log.debug("auto_instrument! called (only: #{only.inspect}, except: #{except.inspect})")
|
|
121
|
+
else
|
|
122
|
+
Braintrust::Log.debug("auto_instrument! called")
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
targets = registry.available
|
|
126
|
+
Braintrust::Log.debug("auto_instrument! available: #{targets.map(&:integration_name).inspect}")
|
|
127
|
+
targets = targets.select { |i| only.include?(i.integration_name) } if only
|
|
128
|
+
targets = targets.reject { |i| except.include?(i.integration_name) } if except
|
|
129
|
+
|
|
130
|
+
results = targets.each_with_object([]) do |integration, instrumented|
|
|
131
|
+
result = instrument!(integration.integration_name)
|
|
132
|
+
# Note: false means skipped (not applicable) or failed (error logged at lower level)
|
|
133
|
+
Braintrust::Log.debug("auto_instrument! :#{integration.integration_name} #{result ? "instrumented" : "skipped"}")
|
|
134
|
+
instrumented << integration.integration_name if result
|
|
135
|
+
end
|
|
136
|
+
Braintrust::Log.debug("auto_instrument! complete: #{results.inspect}")
|
|
137
|
+
results
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Instrument a registered integration by name.
|
|
141
|
+
#
|
|
142
|
+
# @param name [Symbol] The integration name (e.g., :openai, :anthropic)
|
|
143
|
+
# @param options [Hash] Optional configuration
|
|
144
|
+
# @option options [Object] :target Optional target instance to instrument specifically
|
|
145
|
+
# @option options [OpenTelemetry::SDK::Trace::TracerProvider] :tracer_provider Optional tracer provider
|
|
146
|
+
# @return [Boolean] true if instrumentation succeeded
|
|
147
|
+
#
|
|
148
|
+
# @example Instrument all OpenAI clients
|
|
149
|
+
# Braintrust::Contrib.instrument!(:openai)
|
|
150
|
+
#
|
|
151
|
+
# @example Instrument specific OpenAI client instance
|
|
152
|
+
# client = OpenAI::Client.new
|
|
153
|
+
# Braintrust::Contrib.instrument!(:openai, target: client, tracer_provider: my_provider)
|
|
154
|
+
def instrument!(name, **options)
|
|
155
|
+
if (integration = registry[name])
|
|
156
|
+
integration.instrument!(**options)
|
|
157
|
+
else
|
|
158
|
+
Braintrust::Log.error("No integration for '#{name}' is defined!")
|
|
159
|
+
false
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Get the default tracer provider, falling back to OpenTelemetry global.
|
|
164
|
+
# @return [OpenTelemetry::Trace::TracerProvider]
|
|
165
|
+
def default_tracer_provider
|
|
166
|
+
@default_tracer_provider || ::OpenTelemetry.tracer_provider
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
# Get the context for a target object.
|
|
170
|
+
# @param target [Object] The object to retrieve context from
|
|
171
|
+
# @return [Context, nil] The context if found, nil otherwise
|
|
172
|
+
def context_for(target)
|
|
173
|
+
Context.from(target)
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# Get the tracer provider for a target.
|
|
177
|
+
# Checks target's context first, then falls back to contrib default.
|
|
178
|
+
# @param target [Object] The object to look up tracer provider for
|
|
179
|
+
# @return [OpenTelemetry::Trace::TracerProvider]
|
|
180
|
+
def tracer_provider_for(target)
|
|
181
|
+
context_for(target)&.[](:tracer_provider) || default_tracer_provider
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# Get a tracer for a target, using its context's tracer_provider if available.
|
|
185
|
+
# @param target [Object] The object to look up context from
|
|
186
|
+
# @param name [String] Tracer name
|
|
187
|
+
# @return [OpenTelemetry::Trace::Tracer]
|
|
188
|
+
def tracer_for(target, name: "braintrust")
|
|
189
|
+
tracer_provider_for(target).tracer(name)
|
|
190
|
+
end
|
|
191
|
+
end
|
|
192
|
+
end
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
# Load integration stubs (eager load minimal metadata).
|
|
196
|
+
require_relative "contrib/openai/integration"
|
|
197
|
+
require_relative "contrib/ruby_openai/integration"
|
|
198
|
+
require_relative "contrib/ruby_llm/integration"
|
|
199
|
+
require_relative "contrib/anthropic/integration"
|
|
200
|
+
|
|
201
|
+
# Register integrations
|
|
202
|
+
Braintrust::Contrib::OpenAI::Integration.register!
|
|
203
|
+
Braintrust::Contrib::RubyOpenAI::Integration.register!
|
|
204
|
+
Braintrust::Contrib::RubyLLM::Integration.register!
|
|
205
|
+
Braintrust::Contrib::Anthropic::Integration.register!
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Braintrust
|
|
4
|
+
module Internal
|
|
5
|
+
# Environment variable utilities.
|
|
6
|
+
module Env
|
|
7
|
+
ENV_AUTO_INSTRUMENT = "BRAINTRUST_AUTO_INSTRUMENT"
|
|
8
|
+
ENV_INSTRUMENT_EXCEPT = "BRAINTRUST_INSTRUMENT_EXCEPT"
|
|
9
|
+
ENV_INSTRUMENT_ONLY = "BRAINTRUST_INSTRUMENT_ONLY"
|
|
10
|
+
|
|
11
|
+
def self.auto_instrument
|
|
12
|
+
ENV[ENV_AUTO_INSTRUMENT] != "false"
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def self.instrument_except
|
|
16
|
+
parse_list(ENV_INSTRUMENT_EXCEPT)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def self.instrument_only
|
|
20
|
+
parse_list(ENV_INSTRUMENT_ONLY)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Parse a comma-separated environment variable into an array of symbols.
|
|
24
|
+
# @param key [String] The environment variable name
|
|
25
|
+
# @return [Array<Symbol>, nil] Array of symbols, or nil if not set
|
|
26
|
+
def self.parse_list(key)
|
|
27
|
+
value = ENV[key]
|
|
28
|
+
return nil unless value
|
|
29
|
+
value.split(",").map(&:strip).map(&:to_sym)
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Braintrust
|
|
4
|
+
module Internal
|
|
5
|
+
# Time utilities using the monotonic clock for accurate duration measurements.
|
|
6
|
+
#
|
|
7
|
+
# Unlike Time.now, the monotonic clock is not affected by system clock adjustments
|
|
8
|
+
# (NTP updates, daylight saving, manual changes) and provides accurate elapsed time.
|
|
9
|
+
#
|
|
10
|
+
# @see https://blog.dnsimple.com/2018/03/elapsed-time-with-ruby-the-right-way/
|
|
11
|
+
module Time
|
|
12
|
+
# Measure elapsed time using the monotonic clock.
|
|
13
|
+
#
|
|
14
|
+
# Three modes of operation:
|
|
15
|
+
#
|
|
16
|
+
# 1. With a block: executes the block and returns elapsed time in seconds
|
|
17
|
+
# elapsed = Time.measure { some_operation }
|
|
18
|
+
#
|
|
19
|
+
# 2. Without arguments: returns the current monotonic time (for later comparison)
|
|
20
|
+
# start = Time.measure
|
|
21
|
+
# # ... later ...
|
|
22
|
+
# elapsed = Time.measure(start)
|
|
23
|
+
#
|
|
24
|
+
# 3. With a start_time argument: returns elapsed time since start_time
|
|
25
|
+
# start = Time.measure
|
|
26
|
+
# elapsed = Time.measure(start)
|
|
27
|
+
#
|
|
28
|
+
# @param start_time [Float, nil] Optional start time from a previous measure call
|
|
29
|
+
# @yield Optional block to measure
|
|
30
|
+
# @return [Float] Elapsed time in seconds, or current monotonic time if no args/block
|
|
31
|
+
def self.measure(start_time = nil)
|
|
32
|
+
if block_given?
|
|
33
|
+
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
34
|
+
yield
|
|
35
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
|
|
36
|
+
elsif start_time
|
|
37
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
|
|
38
|
+
else
|
|
39
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Setup file for automatic SDK initialization and instrumentation.
|
|
4
|
+
# Load this file to automatically initialize Braintrust and instrument all available LLM libraries.
|
|
5
|
+
#
|
|
6
|
+
# Usage:
|
|
7
|
+
# # Gemfile
|
|
8
|
+
# gem "braintrust", require: "braintrust/setup"
|
|
9
|
+
#
|
|
10
|
+
# # Or in code
|
|
11
|
+
# require "braintrust/setup"
|
|
12
|
+
#
|
|
13
|
+
# Environment variables:
|
|
14
|
+
# BRAINTRUST_API_KEY - Required for tracing to work
|
|
15
|
+
# BRAINTRUST_AUTO_INSTRUMENT - Set to "false" to disable (default: true)
|
|
16
|
+
# BRAINTRUST_INSTRUMENT_ONLY - Comma-separated whitelist
|
|
17
|
+
# BRAINTRUST_INSTRUMENT_EXCEPT - Comma-separated blacklist
|
|
18
|
+
|
|
19
|
+
require_relative "../braintrust"
|
|
20
|
+
require_relative "contrib/setup"
|
|
21
|
+
|
|
22
|
+
module Braintrust
|
|
23
|
+
module Setup
|
|
24
|
+
class << self
|
|
25
|
+
def run!
|
|
26
|
+
return if @setup_complete
|
|
27
|
+
|
|
28
|
+
@setup_complete = true
|
|
29
|
+
|
|
30
|
+
Braintrust::Log.debug("Braintrust setting up...")
|
|
31
|
+
|
|
32
|
+
# Initialize Braintrust (silent failure if no API key)
|
|
33
|
+
# Must run in every process - tracer provider doesn't persist across Kernel.exec
|
|
34
|
+
begin
|
|
35
|
+
Braintrust.init
|
|
36
|
+
rescue => e
|
|
37
|
+
Braintrust::Log.error("Failed to automatically setup Braintrust: #{e.message}")
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Always setup contrib - hooks don't persist across Kernel.exec
|
|
41
|
+
Contrib::Setup.run!
|
|
42
|
+
|
|
43
|
+
Braintrust::Log.debug("Braintrust setup complete. Auto-instrumentation enabled: #{Braintrust::Internal::Env.auto_instrument}")
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Auto-setup when required
|
|
50
|
+
Braintrust::Setup.run!
|
data/lib/braintrust/state.rb
CHANGED
|
@@ -93,6 +93,11 @@ module Braintrust
|
|
|
93
93
|
if enable_tracing
|
|
94
94
|
require_relative "trace"
|
|
95
95
|
Trace.setup(self, tracer_provider, exporter: exporter)
|
|
96
|
+
|
|
97
|
+
# Propagate tracer_provider to Contrib if loaded (soft dependency check)
|
|
98
|
+
if defined?(Braintrust::Contrib)
|
|
99
|
+
Braintrust::Contrib.init(tracer_provider: tracer_provider)
|
|
100
|
+
end
|
|
96
101
|
end
|
|
97
102
|
end
|
|
98
103
|
|
data/lib/braintrust/trace.rb
CHANGED
|
@@ -6,57 +6,6 @@ require_relative "trace/span_processor"
|
|
|
6
6
|
require_relative "trace/span_filter"
|
|
7
7
|
require_relative "logger"
|
|
8
8
|
|
|
9
|
-
# OpenAI integrations - both ruby-openai and openai gems use require "openai"
|
|
10
|
-
# so we detect which one actually loaded the code and require the appropriate integration
|
|
11
|
-
begin
|
|
12
|
-
require "openai"
|
|
13
|
-
|
|
14
|
-
# Check which OpenAI gem's code is actually loaded by inspecting $LOADED_FEATURES
|
|
15
|
-
# (both gems can be in Gem.loaded_specs, but only one's code can be loaded)
|
|
16
|
-
openai_load_path = $LOADED_FEATURES.find { |f| f.end_with?("/openai.rb") }
|
|
17
|
-
|
|
18
|
-
if openai_load_path&.include?("ruby-openai")
|
|
19
|
-
# alexrudall/ruby-openai gem (path contains "ruby-openai-X.Y.Z")
|
|
20
|
-
require_relative "trace/contrib/github.com/alexrudall/ruby-openai/ruby-openai"
|
|
21
|
-
elsif openai_load_path&.include?("/openai-")
|
|
22
|
-
# Official openai gem (path contains "openai-X.Y.Z")
|
|
23
|
-
require_relative "trace/contrib/openai"
|
|
24
|
-
elsif Gem.loaded_specs["ruby-openai"]
|
|
25
|
-
# Fallback: ruby-openai in loaded_specs (for unusual installation paths)
|
|
26
|
-
require_relative "trace/contrib/github.com/alexrudall/ruby-openai/ruby-openai"
|
|
27
|
-
elsif Gem.loaded_specs["openai"]
|
|
28
|
-
# Fallback: official openai in loaded_specs (for unusual installation paths)
|
|
29
|
-
require_relative "trace/contrib/openai"
|
|
30
|
-
end
|
|
31
|
-
rescue LoadError
|
|
32
|
-
# No OpenAI gem installed - integration will not be available
|
|
33
|
-
end
|
|
34
|
-
|
|
35
|
-
# Anthropic integration is optional - automatically loaded if anthropic gem is available
|
|
36
|
-
begin
|
|
37
|
-
require "anthropic"
|
|
38
|
-
require_relative "trace/contrib/anthropic"
|
|
39
|
-
rescue LoadError
|
|
40
|
-
# Anthropic gem not installed - integration will not be available
|
|
41
|
-
end
|
|
42
|
-
|
|
43
|
-
# RubyLLM integration is optional - automatically loaded if ruby_llm gem is available
|
|
44
|
-
#
|
|
45
|
-
# Usage:
|
|
46
|
-
# # Wrap the class once (affects all instances):
|
|
47
|
-
# Braintrust::Trace::RubyLLM.wrap
|
|
48
|
-
#
|
|
49
|
-
# # Or wrap a specific instance:
|
|
50
|
-
# chat = RubyLLM.chat(model: "gpt-4o-mini")
|
|
51
|
-
# Braintrust::Trace::RubyLLM.wrap(chat)
|
|
52
|
-
#
|
|
53
|
-
begin
|
|
54
|
-
require "ruby_llm"
|
|
55
|
-
require_relative "trace/contrib/github.com/crmne/ruby_llm"
|
|
56
|
-
rescue LoadError
|
|
57
|
-
# RubyLLM gem not installed - integration will not be available
|
|
58
|
-
end
|
|
59
|
-
|
|
60
9
|
module Braintrust
|
|
61
10
|
module Trace
|
|
62
11
|
# Set up OpenTelemetry tracing with Braintrust
|
data/lib/braintrust/version.rb
CHANGED