braintrust 0.0.1.alpha.2

data/lib/braintrust/trace/openai.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require "opentelemetry/sdk"
+require "json"
+
+module Braintrust
+  module Trace
+    module OpenAI
+      # Wrap an OpenAI::Client to automatically create spans for chat completions
+      # @param client [OpenAI::Client] the OpenAI client to wrap
+      # @param tracer_provider [OpenTelemetry::SDK::Trace::TracerProvider] the tracer provider (defaults to global)
+      def self.wrap(client, tracer_provider: nil)
+        tracer_provider ||= ::OpenTelemetry.tracer_provider
+
+        # Create a wrapper module that intercepts chat.completions.create
+        wrapper = Module.new do
+          define_method(:create) do |**params|
+            tracer = tracer_provider.tracer("braintrust")
+
+            tracer.in_span("openai.chat.completions.create") do |span|
+              # Initialize metadata hash
+              metadata = {
+                "provider" => "openai",
+                "endpoint" => "/v1/chat/completions"
+              }
+
+              # Capture request metadata fields
+              metadata_fields = %i[
+                model frequency_penalty logit_bias logprobs max_tokens n
+                presence_penalty response_format seed service_tier stop
+                stream stream_options temperature top_p top_logprobs
+                tools tool_choice parallel_tool_calls user functions function_call
+              ]
+
+              metadata_fields.each do |field|
+                metadata[field.to_s] = params[field] if params.key?(field)
+              end
+
+              # Set input messages as JSON
+              if params[:messages]
+                messages_array = params[:messages].map do |msg|
+                  {role: msg[:role].to_s, content: msg[:content]}
+                end
+                span.set_attribute("braintrust.input_json", JSON.generate(messages_array))
+              end
+
+              # Call the original method
+              response = super(**params)
+
+              # Set output (choices) as JSON
+              # Use to_h to get the raw structure with all fields (including tool_calls)
+              if response.respond_to?(:choices) && response.choices&.any?
+                choices_array = response.choices.map(&:to_h)
+                span.set_attribute("braintrust.output_json", JSON.generate(choices_array))
+              end
+
+              # Set metrics (token usage)
+              if response.respond_to?(:usage) && response.usage
+                metrics = {}
+                metrics["prompt_tokens"] = response.usage.prompt_tokens if response.usage.prompt_tokens
+                metrics["completion_tokens"] = response.usage.completion_tokens if response.usage.completion_tokens
+                metrics["tokens"] = response.usage.total_tokens if response.usage.total_tokens
+                span.set_attribute("braintrust.metrics", JSON.generate(metrics))
+              end
+
+              # Add response metadata fields
+              metadata["id"] = response.id if response.respond_to?(:id) && response.id
+              metadata["created"] = response.created if response.respond_to?(:created) && response.created
+              metadata["system_fingerprint"] = response.system_fingerprint if response.respond_to?(:system_fingerprint) && response.system_fingerprint
+              metadata["service_tier"] = response.service_tier if response.respond_to?(:service_tier) && response.service_tier
+
+              # Set metadata ONCE at the end with complete hash
+              span.set_attribute("braintrust.metadata", JSON.generate(metadata))
+
+              response
+            end
+          end
+        end
+
+        # Prepend the wrapper to the completions resource
+        client.chat.completions.singleton_class.prepend(wrapper)
+
+        client
+      end
+    end
+  end
+end

data/lib/braintrust/trace/span_processor.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require "opentelemetry/sdk"
+
+module Braintrust
+  module Trace
+    # Custom span processor that adds Braintrust-specific attributes to spans
+    class SpanProcessor
+      PARENT_ATTR_KEY = "braintrust.parent"
+      ORG_ATTR_KEY = "braintrust.org"
+      APP_URL_ATTR_KEY = "braintrust.app_url"
+
+      def initialize(wrapped_processor, state)
+        @wrapped = wrapped_processor
+        @state = state
+      end
+
+      def on_start(span, parent_context)
+        # Add default parent if span doesn't already have one
+        has_parent = span.respond_to?(:attributes) && span.attributes&.key?(PARENT_ATTR_KEY)
+
+        unless has_parent
+          # Try to inherit parent from parent span in context
+          parent_value = get_parent_from_context(parent_context) || default_parent
+          span.set_attribute(PARENT_ATTR_KEY, parent_value)
+        end
+
+        # Always add org and app_url
+        span.set_attribute(ORG_ATTR_KEY, @state.org_name) if @state.org_name
+        span.set_attribute(APP_URL_ATTR_KEY, @state.app_url) if @state.app_url
+
+        # Delegate to wrapped processor
+        @wrapped.on_start(span, parent_context)
+      end
+
+      # Called when a span ends
+      def on_finish(span)
+        @wrapped.on_finish(span)
+      end
+
+      # Shutdown the processor
+      def shutdown(timeout: nil)
+        @wrapped.shutdown(timeout: timeout)
+      end
+
+      # Force flush any buffered spans
+      def force_flush(timeout: nil)
+        @wrapped.force_flush(timeout: timeout)
+      end
+
+      private
+
+      def default_parent
+        @state.default_parent || "project_name:ruby-sdk-default-project"
+      end
+
+      # Get parent attribute from parent span in context
+      def get_parent_from_context(parent_context)
+        return nil unless parent_context
+
+        # Get the current span from the context (the parent span)
+        parent_span = OpenTelemetry::Trace.current_span(parent_context)
+        return nil unless parent_span
+        return nil unless parent_span.respond_to?(:attributes)
+
+        # Return the parent attribute from the parent span
+        parent_span.attributes&.[](PARENT_ATTR_KEY)
+      end
+    end
+  end
+end

data/lib/braintrust/trace.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require "opentelemetry/sdk"
+require "opentelemetry/exporter/otlp"
+require_relative "trace/span_processor"
+require_relative "trace/openai"
+require_relative "logger"
+
+module Braintrust
+  module Trace
+    def self.enable(tracer_provider, state: nil, exporter: nil)
+      state ||= Braintrust.current_state
+      raise Error, "No state available" unless state
+
+      # Create OTLP HTTP exporter unless override provided
+      exporter ||= OpenTelemetry::Exporter::OTLP::Exporter.new(
+        endpoint: "#{state.api_url}/otel/v1/traces",
+        headers: {
+          "Authorization" => "Bearer #{state.api_key}"
+        }
+      )
+
+      # Wrap in batch processor
+      batch_processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(exporter)
+
+      # Wrap batch processor in our custom span processor to add Braintrust attributes
+      processor = SpanProcessor.new(batch_processor, state)
+
+      # Register with tracer provider
+      tracer_provider.add_span_processor(processor)
+
+      # Console debug if enabled
+      if ENV["BRAINTRUST_ENABLE_TRACE_CONSOLE_LOG"]
+        console_exporter = OpenTelemetry::SDK::Trace::Export::ConsoleSpanExporter.new
+        console_processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(console_exporter)
+        tracer_provider.add_span_processor(console_processor)
+      end
+
+      self
+    end
+
+    # Generate a permalink URL for a span to view in the Braintrust UI
+    # Returns an empty string if the permalink cannot be generated
+    # @param span [OpenTelemetry::Trace::Span] The span to generate a permalink for
+    # @return [String] The permalink URL, or empty string if an error occurs
+    def self.permalink(span)
+      return "" if span.nil?
+
+      # Extract required attributes from span
+      span_context = span.context
+      trace_id = span_context.hex_trace_id
+      span_id = span_context.hex_span_id
+
+      # Get Braintrust attributes
+      attributes = span.attributes if span.respond_to?(:attributes)
+      unless attributes
+        Log.error("Span does not support attributes")
+        return ""
+      end
+
+      app_url = attributes[SpanProcessor::APP_URL_ATTR_KEY]
+      org_name = attributes[SpanProcessor::ORG_ATTR_KEY]
+      parent = attributes[SpanProcessor::PARENT_ATTR_KEY]
+
+      # Validate required attributes
+      unless app_url
+        Log.error("Missing required attribute: #{SpanProcessor::APP_URL_ATTR_KEY}")
+        return ""
+      end
+
+      unless org_name
+        Log.error("Missing required attribute: #{SpanProcessor::ORG_ATTR_KEY}")
+        return ""
+      end
+
+      unless parent
+        Log.error("Missing required attribute: #{SpanProcessor::PARENT_ATTR_KEY}")
+        return ""
+      end
+
+      # Parse parent to determine URL format
+      parent_type, parent_id = parent.split(":", 2)
+      unless parent_type && parent_id
+        Log.error("Invalid parent format: #{parent}")
+        return ""
+      end
+
+      # Build the permalink URL based on parent type
+      if parent_type == "experiment_id"
+        # For experiments: {app_url}/app/{org}/p/{project}/experiments/{experiment_id}?r={trace_id}&s={span_id}
+        project_name, experiment_id = parent_id.split("/", 2)
+        unless project_name && experiment_id
+          Log.error("Invalid experiment parent format: #{parent_id}")
+          return ""
+        end
+
+        "#{app_url}/app/#{org_name}/p/#{project_name}/experiments/#{experiment_id}?r=#{trace_id}&s=#{span_id}"
+      else
+        # For projects: {app_url}/app/{org}/p/{project}/logs?r={trace_id}&s={span_id}
+        # parent_type is typically "project_name"
+        "#{app_url}/app/#{org_name}/p/#{parent_id}/logs?r=#{trace_id}&s=#{span_id}"
+      end
+    rescue => e
+      Log.error("Failed to generate permalink: #{e.message}")
+      ""
+    end
+  end
+end

data/lib/braintrust/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Braintrust
+  VERSION = "0.0.1.alpha.2"
+end

data/lib/braintrust.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require_relative "braintrust/version"
+require_relative "braintrust/config"
+require_relative "braintrust/state"
+require_relative "braintrust/trace"
+require_relative "braintrust/api"
+require_relative "braintrust/internal/experiments"
+require_relative "braintrust/eval"
+
+# Braintrust Ruby SDK
+#
+# OpenTelemetry-based SDK for Braintrust with tracing, OpenAI integration, and evals.
+#
+# @example Initialize with global state
+#   Braintrust.init(
+#     api_key: ENV['BRAINTRUST_API_KEY'],
+#     project: "my-project"
+#   )
+#
+# @example Initialize with explicit state
+#   state = Braintrust.init(
+#     api_key: ENV['BRAINTRUST_API_KEY'],
+#     set_global: false
+#   )
+module Braintrust
+  class Error < StandardError; end
+
+  # Initialize Braintrust SDK
+  # Creates a State from config (ENV + options) and optionally sets it as global
+  #
+  # By default, kicks off an async background login that retries indefinitely.
+  # Use blocking_login: true to login synchronously before returning.
+  #
+  # @param set_global [Boolean] whether to set as global state (default: true)
+  # @param blocking_login [Boolean] whether to block and login synchronously (default: false, which starts async login)
+  # @param tracing [Boolean] whether to enable OpenTelemetry tracing (default: true)
+  # @param tracer_provider [TracerProvider, nil] Optional tracer provider to use instead of creating one
+  # @param api_key [String, nil] Braintrust API key (overrides BRAINTRUST_API_KEY env var)
+  # @param org_name [String, nil] Organization name (overrides BRAINTRUST_ORG_NAME env var)
+  # @param default_parent [String, nil] Default parent for spans (overrides BRAINTRUST_DEFAULT_PROJECT env var, format: "project_name:my-project" or "project_id:uuid")
+  # @param app_url [String, nil] App URL (overrides BRAINTRUST_APP_URL env var, default: https://www.braintrust.dev)
+  # @param api_url [String, nil] API URL (overrides BRAINTRUST_API_URL env var, default: https://api.braintrust.dev)
+  # @return [State] the created state
+  def self.init(set_global: true, blocking_login: false, tracing: true, tracer_provider: nil, **options)
+    config = Config.from_env(**options)
+    state = State.new(
+      api_key: config.api_key,
+      org_name: config.org_name,
+      default_parent: config.default_parent,
+      app_url: config.app_url,
+      api_url: config.api_url
+    )
+
+    State.global = state if set_global
+
+    # Login: either blocking (synchronous) or async (background thread with retries)
+    if blocking_login
+      state.login
+    else
+      state.login_in_thread  # Default: async background login
+    end
+
+    setup_tracing(state, tracer_provider) if tracing
+
+    state
+  end
+
+  # Get the current global state
+  # @return [State, nil] the global state, or nil if not set
+  def self.current_state
+    State.global
+  end
+
+  class << self
+    private
+
+    # Set up OpenTelemetry tracing with Braintrust
+    # @param state [State] Braintrust state
+    # @param explicit_provider [TracerProvider, nil] Optional explicit tracer provider
+    # @return [void]
+    def setup_tracing(state, explicit_provider = nil)
+      require "opentelemetry/sdk"
+
+      if explicit_provider
+        # Use the explicitly provided tracer provider
+        # DO NOT set as global - user is managing it themselves
+        Log.debug("Using explicitly provided OpenTelemetry tracer provider")
+        tracer_provider = explicit_provider
+      else
+        # Check if global tracer provider is already a real TracerProvider
+        current_provider = OpenTelemetry.tracer_provider
+
+        if current_provider.is_a?(OpenTelemetry::SDK::Trace::TracerProvider)
+          # Use existing provider
+          Log.debug("Using existing OpenTelemetry tracer provider")
+          tracer_provider = current_provider
+        else
+          # Create new provider and set as global
+          tracer_provider = OpenTelemetry::SDK::Trace::TracerProvider.new
+          OpenTelemetry.tracer_provider = tracer_provider
+          Log.debug("Created OpenTelemetry tracer provider")
+        end
+      end
+
+      # Enable Braintrust tracing (adds span processor)
+      Trace.enable(tracer_provider, state: state)
+    end
+  end
+end

metadata

@@ -0,0 +1,176 @@
+--- !ruby/object:Gem::Specification
+name: braintrust
+version: !ruby/object:Gem::Version
+  version: 0.0.1.alpha.2
+platform: ruby
+authors:
+- Braintrust
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: opentelemetry-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: opentelemetry-exporter-otlp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.28'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.28'
+- !ruby/object:Gem::Dependency
+  name: openssl
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.3.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.3.1
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: standard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.22'
+- !ruby/object:Gem::Dependency
+  name: openai
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.34'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.34'
+description: OpenTelemetry-based SDK for Braintrust with tracing, OpenAI integration,
+  and evals
+email:
+- info@braintrust.dev
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/braintrust.rb
+- lib/braintrust/api.rb
+- lib/braintrust/api/datasets.rb
+- lib/braintrust/api/functions.rb
+- lib/braintrust/api/internal/auth.rb
+- lib/braintrust/config.rb
+- lib/braintrust/eval.rb
+- lib/braintrust/eval/case.rb
+- lib/braintrust/eval/cases.rb
+- lib/braintrust/eval/functions.rb
+- lib/braintrust/eval/result.rb
+- lib/braintrust/eval/scorer.rb
+- lib/braintrust/internal/experiments.rb
+- lib/braintrust/logger.rb
+- lib/braintrust/state.rb
+- lib/braintrust/trace.rb
+- lib/braintrust/trace/openai.rb
+- lib/braintrust/trace/span_processor.rb
+- lib/braintrust/version.rb
+homepage: https://github.com/braintrustdata/braintrust-sdk-ruby
+licenses:
+- Apache-2.0
+metadata:
+  homepage_uri: https://github.com/braintrustdata/braintrust-sdk-ruby
+  source_code_uri: https://github.com/braintrustdata/braintrust-sdk-ruby
+  changelog_uri: https://github.com/braintrustdata/braintrust-sdk-ruby/blob/main/CHANGELOG.md
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Ruby SDK for Braintrust
+test_files: []