braintrust 0.1.0 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b28ce15a3d9599baecd29de264cd9455811cc18b1338fb948a19347f3c6327cc
-  data.tar.gz: f752b82b8f79ec77f72b2b95b5c98f77b5de92af77d39370eccc87af4a2835d7
+  metadata.gz: 626876b443795d28b4ba5d12f8bf10381c3052d5d196adb01207d545303f3d1e
+  data.tar.gz: 347ca89ea9f485ca6521a38c067bdd15074db4e6a4757523901888a8d4cc3e9c
 SHA512:
-  metadata.gz: 86b922e9252bb1e390e892458931b4fc19d495ba04842f54f37d3a0fdb52db02fb7e5237bee65df74f870fa457cfefa98d46abe65bdf855ce46bdfe7056d9c97
-  data.tar.gz: 7f426e8c342d57136c175dae2c69359a8103da8ec7425750c3bc86de0232ac505b78f2d82c547124070e8e7cacd6cd9c479e45b74ad06b6379118af1eb281cb8
+  metadata.gz: 7b827a4f92e2bc4b39e41174e62dacdc431fdc0b6c8d13882bdcaaa369af9621174fc01bafa3d0d594b71464ea374c6904e9834631957951dad87d6583a58dc9
+  data.tar.gz: bb6f2d3807765ef4ad591849e0972379fc3f97ef8d90bda0785e1d4dab87ce5e91d954d9d3c8fc7eff6c9295d120d6cbe07acb5bb348873c842d791a3fbdce84

data/README.md

@@ -136,6 +136,7 @@ Braintrust.init(
 | `BRAINTRUST_AUTO_INSTRUMENT`      | Set to `false` to disable auto-instrumentation                            |
 | `BRAINTRUST_DEBUG`                | Set to `true` to enable debug logging                                     |
 | `BRAINTRUST_DEFAULT_PROJECT`      | Default project for spans                                                 |
+| `BRAINTRUST_FLUSH_ON_EXIT`        | Set to `false` to disable automatic span flushing on program exit         |
 | `BRAINTRUST_INSTRUMENT_EXCEPT`    | Comma-separated list of integrations to skip                              |
 | `BRAINTRUST_INSTRUMENT_ONLY`      | Comma-separated list of integrations to enable (e.g., `openai,anthropic`) |
 | `BRAINTRUST_ORG_NAME`             | Organization name                                                         |

data/lib/braintrust/api/functions.rb

@@ -85,6 +85,17 @@ module Braintrust
         http_post_json("/v1/function/#{id}/invoke", payload)
       end
 
+      # Get a function by ID (includes full prompt_data)
+      # GET /v1/function/{id}
+      # @param id [String] Function UUID
+      # @param version [String, nil] Retrieve prompt at a specific version (transaction ID or version identifier)
+      # @return [Hash] Full function data including prompt_data
+      def get(id:, version: nil)
+        params = {}
+        params["version"] = version if version
+        http_get("/v1/function/#{id}", params)
+      end
+
       # Delete a function by ID
       # DELETE /v1/function/{id}
       # @param id [String] Function UUID

data/lib/braintrust/contrib/anthropic/instrumentation/beta_messages.rb

@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+require "opentelemetry/sdk"
+require "json"
+require_relative "../../support/otel"
+require_relative "common"
+require_relative "../../../internal/time"
+
+module Braintrust
+  module Contrib
+    module Anthropic
+      module Instrumentation
+        # Beta Messages instrumentation for Anthropic.
+        # Wraps client.beta.messages.create() and stream() methods to create spans.
+        #
+        # @note Beta APIs are experimental and subject to change between SDK versions.
+        #   This module includes defensive coding to handle response format changes.
+        module BetaMessages
+          def self.included(base)
+            base.prepend(InstanceMethods) unless applied?(base)
+          end
+
+          def self.applied?(base)
+            base.ancestors.include?(InstanceMethods)
+          end
+
+          module InstanceMethods
+            # Standard metadata fields (shared with stable API)
+            METADATA_FIELDS = %i[
+              model max_tokens temperature top_p top_k stop_sequences
+              stream tools tool_choice thinking metadata service_tier
+            ].freeze
+
+            # Beta-specific metadata fields
+            BETA_METADATA_FIELDS = %i[
+              betas output_format
+            ].freeze
+
+            # Wrap synchronous beta.messages.create
+            def create(**params)
+              client = instance_variable_get(:@client)
+              tracer = Braintrust::Contrib.tracer_for(client)
+
+              tracer.in_span("anthropic.messages.create") do |span|
+                # Pre-call instrumentation (swallow errors)
+                metadata = nil
+                begin
+                  metadata = build_metadata(params)
+                  set_input(span, params)
+                rescue => e
+                  Braintrust::Log.debug("Beta API: Failed to capture request: #{e.message}")
+                  metadata ||= {"provider" => "anthropic", "api_version" => "beta"}
+                end
+
+                # API call - let errors propagate naturally
+                response = nil
+                time_to_first_token = Braintrust::Internal::Time.measure do
+                  response = super(**params)
+                end
+
+                # Post-call instrumentation (swallow errors)
+                begin
+                  set_output(span, response)
+                  set_metrics(span, response, time_to_first_token)
+                  finalize_metadata(span, metadata, response)
+                rescue => e
+                  Braintrust::Log.debug("Beta API: Failed to capture response: #{e.message}")
+                end
+
+                response
+              end
+            end
+
+            # Wrap streaming beta.messages.stream
+            # Stores context on stream object for span creation during consumption
+            def stream(**params)
+              client = instance_variable_get(:@client)
+              tracer = Braintrust::Contrib.tracer_for(client)
+
+              # Pre-call instrumentation (swallow errors)
+              metadata = nil
+              begin
+                metadata = build_metadata(params, stream: true)
+              rescue => e
+                Braintrust::Log.debug("Beta API: Failed to build stream metadata: #{e.message}")
+                metadata = {"provider" => "anthropic", "api_version" => "beta", "stream" => true}
+              end
+
+              # API call - let errors propagate naturally
+              stream_obj = super
+
+              # Post-call instrumentation (swallow errors)
+              begin
+                Braintrust::Contrib::Context.set!(stream_obj,
+                  tracer: tracer,
+                  params: params,
+                  metadata: metadata,
+                  messages_instance: self,
+                  start_time: Braintrust::Internal::Time.measure)
+              rescue => e
+                Braintrust::Log.debug("Beta API: Failed to set stream context: #{e.message}")
+              end
+
+              stream_obj
+            end
+
+            private
+
+            def finalize_stream_span(span, stream_obj, metadata, time_to_first_token)
+              if stream_obj.respond_to?(:accumulated_message)
+                begin
+                  msg = stream_obj.accumulated_message
+                  set_output(span, msg)
+                  set_metrics(span, msg, time_to_first_token)
+                  metadata["stop_reason"] = msg.stop_reason if msg.respond_to?(:stop_reason) && msg.stop_reason
+                  metadata["model"] = msg.model if msg.respond_to?(:model) && msg.model
+                rescue => e
+                  Braintrust::Log.debug("Beta API: Failed to get accumulated message: #{e.message}")
+                end
+              end
+              Support::OTel.set_json_attr(span, "braintrust.metadata", metadata)
+            end
+
+            def build_metadata(params, stream: false)
+              metadata = {
+                "provider" => "anthropic",
+                "endpoint" => "/v1/messages",
+                "api_version" => "beta"
+              }
+              metadata["stream"] = true if stream
+
+              # Capture standard fields
+              METADATA_FIELDS.each do |field|
+                metadata[field.to_s] = params[field] if params.key?(field)
+              end
+
+              # Capture beta-specific fields with defensive handling
+              capture_beta_fields(metadata, params)
+
+              metadata
+            rescue => e
+              Braintrust::Log.debug("Beta API: Failed to build metadata: #{e.message}")
+              {"provider" => "anthropic", "api_version" => "beta"}
+            end
+
+            def capture_beta_fields(metadata, params)
+              # Capture betas array (e.g., ["structured-outputs-2025-11-13"])
+              if params.key?(:betas)
+                betas = params[:betas]
+                metadata["betas"] = betas.is_a?(Array) ? betas : [betas]
+              end
+
+              # Capture output_format for structured outputs
+              if params.key?(:output_format)
+                output_format = params[:output_format]
+                metadata["output_format"] = begin
+                  if output_format.respond_to?(:to_h)
+                    output_format.to_h
+                  else
+                    output_format
+                  end
+                rescue
+                  output_format.to_s
+                end
+              end
+            end
+
+            def set_input(span, params)
+              input_messages = []
+
+              begin
+                if params[:system]
+                  system_content = params[:system]
+                  if system_content.is_a?(Array)
+                    system_text = system_content.map { |blk|
+                      blk.is_a?(Hash) ? blk[:text] : blk
+                    }.join("\n")
+                    input_messages << {role: "system", content: system_text}
+                  else
+                    input_messages << {role: "system", content: system_content}
+                  end
+                end
+
+                if params[:messages]
+                  messages_array = params[:messages].map { |m| m.respond_to?(:to_h) ? m.to_h : m }
+                  input_messages.concat(messages_array)
+                end
+
+                Support::OTel.set_json_attr(span, "braintrust.input_json", input_messages) if input_messages.any?
+              rescue => e
+                Braintrust::Log.debug("Beta API: Failed to capture input: #{e.message}")
+              end
+            end
+
+            def set_output(span, response)
+              return unless response
+
+              begin
+                return unless response.respond_to?(:content) && response.content
+
+                content_array = response.content.map { |c| c.respond_to?(:to_h) ? c.to_h : c }
+                output = [{
+                  role: response.respond_to?(:role) ? response.role : "assistant",
+                  content: content_array
+                }]
+                Support::OTel.set_json_attr(span, "braintrust.output_json", output)
+              rescue => e
+                Braintrust::Log.debug("Beta API: Failed to capture output: #{e.message}")
+              end
+            end
+
+            def set_metrics(span, response, time_to_first_token)
+              metrics = {}
+
+              begin
+                if response.respond_to?(:usage) && response.usage
+                  metrics = Common.parse_usage_tokens(response.usage)
+                end
+                metrics["time_to_first_token"] = time_to_first_token if time_to_first_token
+                Support::OTel.set_json_attr(span, "braintrust.metrics", metrics) unless metrics.empty?
+              rescue => e
+                Braintrust::Log.debug("Beta API: Failed to capture metrics: #{e.message}")
+              end
+            end
+
+            def finalize_metadata(span, metadata, response)
+              begin
+                metadata["stop_reason"] = response.stop_reason if response.respond_to?(:stop_reason) && response.stop_reason
+                metadata["stop_sequence"] = response.stop_sequence if response.respond_to?(:stop_sequence) && response.stop_sequence
+                metadata["model"] = response.model if response.respond_to?(:model) && response.model
+              rescue => e
+                Braintrust::Log.debug("Beta API: Failed to finalize metadata: #{e.message}")
+              end
+
+              Support::OTel.set_json_attr(span, "braintrust.metadata", metadata)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/contrib/anthropic/integration.rb

@@ -40,12 +40,12 @@ module Braintrust
           defined?(::Anthropic::Client) ? true : false
         end
 
-        # Lazy-load the patcher only when actually patching.
+        # Lazy-load the patchers only when actually patching.
         # This keeps the integration stub lightweight.
         # @return [Array<Class>] The patcher classes
         def self.patchers
           require_relative "patcher"
-          [MessagesPatcher]
+          [MessagesPatcher, BetaMessagesPatcher]
         end
       end
     end

data/lib/braintrust/contrib/anthropic/patcher.rb

@@ -2,6 +2,7 @@
 
 require_relative "../patcher"
 require_relative "instrumentation/messages"
+require_relative "instrumentation/beta_messages"
 
 module Braintrust
   module Contrib
@@ -57,6 +58,88 @@ module Braintrust
           end
         end
       end
+
+      # Patcher for Anthropic beta messages API.
+      # Instruments client.beta.messages.create and stream methods.
+      #
+      # @note Beta APIs are experimental and subject to change between SDK versions.
+      #   Braintrust will make reasonable efforts to maintain compatibility, but
+      #   breaking changes may require SDK updates.
+      #
+      # @see https://docs.anthropic.com/en/docs/build-with-claude/structured-outputs
+      #   for structured outputs documentation
+      class BetaMessagesPatcher < Braintrust::Contrib::Patcher
+        # Version constraints for beta patcher.
+        # Set MAXIMUM_VERSION when a breaking change is discovered to disable
+        # beta instrumentation on incompatible versions until a fix is released.
+        # Currently nil = rely on class existence check only.
+        MAXIMUM_VERSION = nil
+
+        class << self
+          def applicable?
+            return false unless defined?(::Anthropic::Client)
+            return false unless defined?(::Anthropic::Resources::Beta::Messages)
+            return false if MAXIMUM_VERSION && !version_compatible?
+            true
+          end
+
+          def patched?(**options)
+            target_class = get_singleton_class(options[:target]) || ::Anthropic::Resources::Beta::Messages
+            Instrumentation::BetaMessages.applied?(target_class)
+          end
+
+          # Perform the actual patching.
+          # @param options [Hash] Configuration options passed from integration
+          # @option options [Object] :target Optional target instance to patch
+          # @option options [OpenTelemetry::SDK::Trace::TracerProvider] :tracer_provider Optional tracer provider
+          # @return [void]
+          def perform_patch(**options)
+            return unless applicable?
+
+            Braintrust::Log.debug("Instrumenting Anthropic beta.messages API (experimental)")
+
+            # MessageStream is shared with stable API - already patched by MessagesPatcher.
+            # The BetaMessages instrumentation sets api_version: "beta" in context,
+            # which MessageStream uses to include in metadata.
+            patch_message_stream
+
+            if options[:target]
+              # Instance-level (for only this client instance)
+              raise ArgumentError, "target must be a kind of ::Anthropic::Client" unless options[:target].is_a?(::Anthropic::Client)
+
+              get_singleton_class(options[:target]).include(Instrumentation::BetaMessages)
+            else
+              # Class-level (for all client instances)
+              ::Anthropic::Resources::Beta::Messages.include(Instrumentation::BetaMessages)
+            end
+          end
+
+          private
+
+          def version_compatible?
+            return true unless MAXIMUM_VERSION
+
+            spec = Gem.loaded_specs["anthropic"]
+            return true unless spec
+
+            spec.version <= Gem::Version.new(MAXIMUM_VERSION)
+          end
+
+          def get_singleton_class(client)
+            client&.beta&.messages&.singleton_class
+          rescue
+            # Defensive: beta namespace may not exist or may have changed
+            nil
+          end
+
+          def patch_message_stream
+            return unless defined?(::Anthropic::Helpers::Streaming::MessageStream)
+            return if Instrumentation::MessageStream.applied?(::Anthropic::Helpers::Streaming::MessageStream)
+
+            ::Anthropic::Helpers::Streaming::MessageStream.include(Instrumentation::MessageStream)
+          end
+        end
+      end
     end
   end
 end

data/lib/braintrust/contrib/openai/instrumentation/moderations.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "opentelemetry/sdk"
+require "json"
+
+require_relative "../../../internal/time"
+require_relative "../../support/otel"
+
+module Braintrust
+  module Contrib
+    module OpenAI
+      module Instrumentation
+        # Moderations API instrumentation for OpenAI.
+        # Wraps create() method to create spans.
+        module Moderations
+          def self.included(base)
+            base.prepend(InstanceMethods) unless applied?(base)
+          end
+
+          def self.applied?(base)
+            base.ancestors.include?(InstanceMethods)
+          end
+
+          METADATA_FIELDS = %i[
+            model
+          ].freeze
+
+          module InstanceMethods
+            # Wrap non-streaming create method
+            def create(**params)
+              client = instance_variable_get(:@client)
+              tracer = Braintrust::Contrib.tracer_for(client)
+
+              tracer.in_span("openai.moderations.create") do |span|
+                metadata = build_metadata(params)
+
+                set_input(span, params)
+
+                response = nil
+                time_to_first_token = Braintrust::Internal::Time.measure do
+                  response = super
+                end
+
+                set_output(span, response)
+                set_metrics(span, time_to_first_token)
+                finalize_metadata(span, metadata, response)
+
+                response
+              end
+            end
+
+            private
+
+            def build_metadata(params)
+              metadata = {
+                "provider" => "openai",
+                "endpoint" => "/v1/moderations"
+              }
+              Moderations::METADATA_FIELDS.each do |field|
+                metadata[field.to_s] = params[field] if params.key?(field)
+              end
+              metadata
+            end
+
+            def set_input(span, params)
+              return unless params[:input]
+
+              Support::OTel.set_json_attr(span, "braintrust.input_json", params[:input])
+            end
+
+            def set_output(span, response)
+              return unless response.respond_to?(:results) && response.results
+
+              Support::OTel.set_json_attr(span, "braintrust.output_json", response.results)
+            end
+
+            def set_metrics(span, time_to_first_token)
+              metrics = {}
+              metrics["time_to_first_token"] = time_to_first_token
+              Support::OTel.set_json_attr(span, "braintrust.metrics", metrics) unless metrics.empty?
+            end
+
+            def finalize_metadata(span, metadata, response)
+              metadata["id"] = response.id if response.respond_to?(:id) && response.id
+              metadata["model"] = response.model if response.respond_to?(:model) && response.model
+              Support::OTel.set_json_attr(span, "braintrust.metadata", metadata)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/contrib/openai/integration.rb

@@ -50,7 +50,7 @@ module Braintrust
         # @return [Class] The patcher class
         def self.patchers
           require_relative "patcher"
-          [ChatPatcher, ResponsesPatcher]
+          [ChatPatcher, ResponsesPatcher, ModerationsPatcher]
         end
       end
     end

data/lib/braintrust/contrib/openai/patcher.rb

@@ -3,6 +3,7 @@
 require_relative "../patcher"
 require_relative "instrumentation/chat"
 require_relative "instrumentation/responses"
+require_relative "instrumentation/moderations"
 
 module Braintrust
   module Contrib
@@ -125,6 +126,48 @@ module Braintrust
           end
         end
       end
+
+      # Patcher for OpenAI Moderations API - implements class-level patching.
+      # All new OpenAI::Client instances created after patch! will be automatically instrumented.
+      class ModerationsPatcher < Braintrust::Contrib::Patcher
+        class << self
+          def applicable?
+            defined?(::OpenAI::Client) && ::OpenAI::Client.instance_methods.include?(:moderations)
+          end
+
+          def patched?(**options)
+            # Use the target's singleton class if provided, otherwise check the base class.
+            target_class = get_singleton_class(options[:target]) || ::OpenAI::Resources::Moderations
+
+            Instrumentation::Moderations.applied?(target_class)
+          end
+
+          # Perform the actual patching.
+          # @param options [Hash] Configuration options passed from integration
+          # @option options [Object] :target Optional target instance to patch
+          # @option options [OpenTelemetry::SDK::Trace::TracerProvider] :tracer_provider Optional tracer provider
+          # @return [void]
+          def perform_patch(**options)
+            return unless applicable?
+
+            if options[:target]
+              # Instance-level (for only this client)
+              raise ArgumentError, "target must be a kind of ::OpenAI::Client" unless options[:target].is_a?(::OpenAI::Client)
+
+              get_singleton_class(options[:target]).include(Instrumentation::Moderations)
+            else
+              # Class-level (for all clients)
+              ::OpenAI::Resources::Moderations.include(Instrumentation::Moderations)
+            end
+          end
+
+          private
+
+          def get_singleton_class(client)
+            client&.moderations&.singleton_class
+          end
+        end
+      end
     end
   end
 end

data/lib/braintrust/contrib/ruby_openai/instrumentation/moderations.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+require "opentelemetry/sdk"
+require "json"
+
+require_relative "../../support/otel"
+require_relative "../../../internal/time"
+
+module Braintrust
+  module Contrib
+    module RubyOpenAI
+      module Instrumentation
+        # Moderations API instrumentation for ruby-openai.
+        # Provides module that can be prepended to OpenAI::Client to instrument the moderations method.
+        module Moderations
+          def self.included(base)
+            base.prepend(InstanceMethods) unless applied?(base)
+          end
+
+          def self.applied?(base)
+            base.ancestors.include?(InstanceMethods)
+          end
+
+          METADATA_FIELDS = %i[
+            model
+          ].freeze
+
+          module InstanceMethods
+            # Wrap moderations method for ruby-openai gem
+            # ruby-openai API: client.moderations(parameters: {...})
+            def moderations(parameters:)
+              tracer = Braintrust::Contrib.tracer_for(self)
+
+              tracer.in_span("openai.moderations.create") do |span|
+                metadata = build_moderations_metadata(parameters)
+                set_moderations_input(span, parameters)
+
+                response = nil
+                time_to_first_token = Braintrust::Internal::Time.measure do
+                  response = super(parameters: parameters)
+                end
+
+                set_moderations_output(span, response)
+                set_moderations_metrics(span, time_to_first_token)
+                finalize_moderations_metadata(span, metadata, response)
+
+                response
+              end
+            end
+
+            private
+
+            def build_moderations_metadata(parameters)
+              metadata = {
+                "provider" => "openai",
+                "endpoint" => "/v1/moderations"
+              }
+
+              Moderations::METADATA_FIELDS.each do |field|
+                metadata[field.to_s] = parameters[field] if parameters.key?(field)
+              end
+
+              metadata
+            end
+
+            def set_moderations_input(span, parameters)
+              return unless parameters[:input]
+              Support::OTel.set_json_attr(span, "braintrust.input_json", parameters[:input])
+            end
+
+            def set_moderations_output(span, response)
+              results = response["results"] || response[:results]
+              return unless results
+              Support::OTel.set_json_attr(span, "braintrust.output_json", results)
+            end
+
+            def set_moderations_metrics(span, time_to_first_token)
+              metrics = {"time_to_first_token" => time_to_first_token}
+              Support::OTel.set_json_attr(span, "braintrust.metrics", metrics)
+            end
+
+            def finalize_moderations_metadata(span, metadata, response)
+              %w[id model].each do |field|
+                value = response[field] || response[field.to_sym]
+                metadata[field] = value if value
+              end
+              Support::OTel.set_json_attr(span, "braintrust.metadata", metadata)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/contrib/ruby_openai/integration.rb

@@ -50,7 +50,7 @@ module Braintrust
         # @return [Array<Class>] The patcher classes
         def self.patchers
           require_relative "patcher"
-          [ChatPatcher, ResponsesPatcher]
+          [ChatPatcher, ResponsesPatcher, ModerationsPatcher]
         end
       end
     end