braintrust 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: be1db72d5636027cfb0b1744e0c30ac9b7169b7f898d4a911639d9f437e0654a
-  data.tar.gz: 725708134bfa2a8d12a3fb0c74e9a54f97afe2f249ed6e3d26184cc5c97a3775
+  metadata.gz: 3330f067e867f2d0d81ea12db80bb2dbf83d9ca447ef819e2bff7c97138e7f9e
+  data.tar.gz: f2e4f8e34bfe2292d5b3883923d4b638de89b6c930144c02a4c7941b831ca7e6
 SHA512:
-  metadata.gz: 85cda56b35738299973d2d58a001c9cb746c168437de56e9cf443a2787728dc98014b2b63cf5d6f0d781b8d5c443feedc42dab5abaae2827b60f1c80eb38329a
-  data.tar.gz: e99f24f4cfabbffbdc21822e35145794ccdf29bd22d2e47889dfaab30b4428b49ae173c3ad3e12c4fc7169e3a61ed493670c55bacacd25a800c5df28dc68269f
+  metadata.gz: feb0969a48253f3729b9f38d243abcb6f0367176b9cddf9df112be9e415f8b5dadf90ca50588bd2e09be84a4efb406f40aa13b4ccab88310bba1a73e993205e6
+  data.tar.gz: 67e9c6163c4b20944953e7063286f98b2e079e424fd80991376110864971f251cb1ca6a663a99d923d9cd92d74be17030dba90a9ba5619f610d307d4075aa38f

data/README.md

@@ -1,6 +1,7 @@
 # Braintrust Ruby SDK
 
 [![Gem Version](https://badge.fury.io/rb/braintrust.svg)](https://badge.fury.io/rb/braintrust)
+[![Documentation](https://img.shields.io/badge/docs-rubydoc.info-blue.svg)](https://rubydoc.info/gems/braintrust)
 ![Beta](https://img.shields.io/badge/status-beta-yellow)
 
 ## Overview
@@ -46,46 +47,24 @@ export BRAINTRUST_API_KEY="your-api-key"
 ```ruby
 require "braintrust"
 
-# Initialize Braintrust
 Braintrust.init
 
-# Simple food classifier (the code being evaluated)
-def classify_food(input)
-  fruit = %w[apple banana strawberry orange grape mango]
-  vegetable = %w[carrot broccoli spinach potato tomato cucumber]
-
-  input_lower = input.downcase
-  return "fruit" if fruit.any? { |f| input_lower.include?(f) }
-  return "vegetable" if vegetable.any? { |v| input_lower.include?(v) }
-  "unknown"
-end
+# Define task to evaluate
+task = ->(input) { input.include?("a") ? "fruit" : "vegetable" }
 
-# Run an evaluation
-result = Braintrust::Eval.run(
+# Run evaluation
+Braintrust::Eval.run(
   project: "my-project",
-  experiment: "food-classifier-eval",
-
-  # Test cases
+  experiment: "food-classifier",
   cases: [
     {input: "apple", expected: "fruit"},
-    {input: "carrot", expected: "vegetable"},
-    {input: "banana", expected: "fruit"},
-    {input: "broccoli", expected: "vegetable"}
+    {input: "carrot", expected: "vegetable"}
   ],
-
-  # Task to evaluate
-  task: ->(input) { classify_food(input) },
-
-  # Scorers to judge output quality
+  task: task,
   scorers: [
-    Braintrust::Eval.scorer("exact_match") { |input, expected, output|
-      (output == expected) ? 1.0 : 0.0
-    }
+    ->(input, expected, output) { output == expected ? 1.0 : 0.0 }
   ]
 )
-
-# View results
-puts "View results at: #{result.permalink}"
 ```
 
 ### Tracing
@@ -128,20 +107,15 @@ puts "View trace in Braintrust!"
 require "braintrust"
 require "openai"
 
-# Initialize Braintrust
 Braintrust.init
 
-# Create OpenAI client
 client = OpenAI::Client.new(api_key: ENV["OPENAI_API_KEY"])
 
-# Wrap the client with Braintrust tracing
 Braintrust::Trace::OpenAI.wrap(client)
 
-# Create a root span to capture the operation
 tracer = OpenTelemetry.tracer_provider.tracer("openai-app")
 root_span = nil
 
-# Make a chat completion request (automatically traced!)
 response = tracer.in_span("chat-completion") do |span|
   root_span = span
 
@@ -157,10 +131,8 @@ end
 
 puts "Response: #{response.choices[0].message.content}"
 
-# View the trace
 puts "View trace at: #{Braintrust::Trace.permalink(root_span)}"
 
-# Shutdown to flush spans
 OpenTelemetry.tracer_provider.shutdown
 ```
 

data/lib/braintrust/config.rb

@@ -16,15 +16,20 @@ module Braintrust
 
     # Create a Config from environment variables, with option overrides
     # Passed-in options take priority over ENV vars
-    def self.from_env(**options)
-      defaults = {
-        api_key: ENV["BRAINTRUST_API_KEY"],
-        org_name: ENV["BRAINTRUST_ORG_NAME"],
-        default_project: ENV["BRAINTRUST_DEFAULT_PROJECT"],
-        app_url: ENV["BRAINTRUST_APP_URL"] || "https://www.braintrust.dev",
-        api_url: ENV["BRAINTRUST_API_URL"] || "https://api.braintrust.dev"
-      }
-      new(**defaults.merge(options))
+    # @param api_key [String, nil] Braintrust API key (overrides BRAINTRUST_API_KEY env var)
+    # @param org_name [String, nil] Organization name (overrides BRAINTRUST_ORG_NAME env var)
+    # @param default_project [String, nil] Default project (overrides BRAINTRUST_DEFAULT_PROJECT env var)
+    # @param app_url [String, nil] App URL (overrides BRAINTRUST_APP_URL env var)
+    # @param api_url [String, nil] API URL (overrides BRAINTRUST_API_URL env var)
+    # @return [Config] the created config
+    def self.from_env(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil)
+      new(
+        api_key: api_key || ENV["BRAINTRUST_API_KEY"],
+        org_name: org_name || ENV["BRAINTRUST_ORG_NAME"],
+        default_project: default_project || ENV["BRAINTRUST_DEFAULT_PROJECT"],
+        app_url: app_url || ENV["BRAINTRUST_APP_URL"] || "https://www.braintrust.dev",
+        api_url: api_url || ENV["BRAINTRUST_API_URL"] || "https://api.braintrust.dev"
+      )
     end
   end
 end

data/lib/braintrust/eval/result.rb

@@ -5,21 +5,23 @@ module Braintrust
     # Result represents the outcome of an evaluation run
     # Contains experiment metadata, errors, and timing information
     class Result
-      attr_reader :experiment_id, :experiment_name, :project_id,
+      attr_reader :experiment_id, :experiment_name, :project_id, :project_name,
         :permalink, :errors, :duration
 
       # Create a new result
       # @param experiment_id [String] The experiment ID
       # @param experiment_name [String] The experiment name
       # @param project_id [String] The project ID
+      # @param project_name [String] The project name
       # @param permalink [String] Link to view the experiment in Braintrust UI
       # @param errors [Array<String>] List of errors that occurred
       # @param duration [Float] Duration in seconds
-      def initialize(experiment_id:, experiment_name:, project_id:,
+      def initialize(experiment_id:, experiment_name:, project_id:, project_name:,
         permalink:, errors:, duration:)
         @experiment_id = experiment_id
         @experiment_name = experiment_name
         @project_id = project_id
+        @project_name = project_name
         @permalink = permalink
         @errors = errors
         @duration = duration
@@ -42,9 +44,10 @@ module Braintrust
       def to_s
         [
           "Experiment: #{experiment_name}",
+          "Project: #{project_name}",
           "ID: #{experiment_id}",
           "Link: #{permalink}",
-          "Duration: #{duration.round(2)}s",
+          "Duration: #{duration.round(4)}s",
           "Errors: #{errors.length}"
         ].join("\n")
       end

data/lib/braintrust/eval.rb

@@ -137,6 +137,7 @@ module Braintrust
           experiment_id: experiment_id,
           experiment_name: experiment_name,
           project_id: project_id,
+          project_name: project_name,
           permalink: permalink,
           errors: errors,
           duration: duration
@@ -146,6 +147,7 @@ module Braintrust
       # Print result summary to stdout
       # @param result [Result] The evaluation result
       def print_result(result)
+        puts "=" * 60
         puts result
       end
 
@@ -295,7 +297,7 @@ module Braintrust
       # @param scorers [Array<Scorer>] The scorers
       # @param errors [Array<String>] Error collection array
       # @param tracer [Tracer] OpenTelemetry tracer
-      # @param parent_attr [String] Parent attribute (experiment_id:project/exp_id)
+      # @param parent_attr [String] Parent attribute (experiment_id:exp_id)
       def run_case(test_case, task, scorers, errors, tracer, parent_attr)
         # Create eval span (parent)
         tracer.in_span("eval") do |eval_span|

data/lib/braintrust/state.rb

@@ -11,7 +11,38 @@ module Braintrust
     @mutex = Mutex.new
     @global_state = nil
 
-    def initialize(api_key: nil, org_name: nil, org_id: nil, default_project: nil, app_url: nil, api_url: nil, proxy_url: nil, logged_in: false)
+    # Create a State from environment variables with option overrides
+    # @param api_key [String, nil] Braintrust API key (overrides BRAINTRUST_API_KEY env var)
+    # @param org_name [String, nil] Organization name (overrides BRAINTRUST_ORG_NAME env var)
+    # @param default_project [String, nil] Default project (overrides BRAINTRUST_DEFAULT_PROJECT env var)
+    # @param app_url [String, nil] App URL (overrides BRAINTRUST_APP_URL env var)
+    # @param api_url [String, nil] API URL (overrides BRAINTRUST_API_URL env var)
+    # @param blocking_login [Boolean] whether to block and login synchronously (default: false)
+    # @param enable_tracing [Boolean] whether to enable OpenTelemetry tracing (default: true)
+    # @param tracer_provider [TracerProvider, nil] Optional tracer provider to use
+    # @return [State] the created state
+    def self.from_env(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil, blocking_login: false, enable_tracing: true, tracer_provider: nil)
+      require_relative "config"
+      config = Config.from_env(
+        api_key: api_key,
+        org_name: org_name,
+        default_project: default_project,
+        app_url: app_url,
+        api_url: api_url
+      )
+      new(
+        api_key: config.api_key,
+        org_name: config.org_name,
+        default_project: config.default_project,
+        app_url: config.app_url,
+        api_url: config.api_url,
+        blocking_login: blocking_login,
+        enable_tracing: enable_tracing,
+        tracer_provider: tracer_provider
+      )
+    end
+
+    def initialize(api_key: nil, org_name: nil, org_id: nil, default_project: nil, app_url: nil, api_url: nil, proxy_url: nil, blocking_login: false, enable_tracing: true, tracer_provider: nil)
       # Instance-level mutex for thread-safe login
       @login_mutex = Mutex.new
       raise ArgumentError, "api_key is required" if api_key.nil? || api_key.empty?
@@ -23,7 +54,20 @@ module Braintrust
       @app_url = app_url || "https://www.braintrust.dev"
       @api_url = api_url
       @proxy_url = proxy_url
-      @logged_in = logged_in
+      @logged_in = false
+
+      # Perform login after state setup
+      if blocking_login
+        login
+      else
+        login_in_thread
+      end
+
+      # Setup tracing if requested
+      if enable_tracing
+        require_relative "trace"
+        Trace.setup(self, tracer_provider)
+      end
     end
 
     # Thread-safe global state getter

data/lib/braintrust/trace.rb

@@ -15,6 +15,35 @@ end
 
 module Braintrust
   module Trace
+    # Set up OpenTelemetry tracing with Braintrust
+    # @param state [State] Braintrust state
+    # @param tracer_provider [TracerProvider, nil] Optional tracer provider
+    # @return [void]
+    def self.setup(state, tracer_provider = nil)
+      if tracer_provider
+        # Use the explicitly provided tracer provider
+        # DO NOT set as global - user is managing it themselves
+        Log.debug("Using explicitly provided OpenTelemetry tracer provider")
+      else
+        # Check if global tracer provider is already a real TracerProvider
+        current_provider = OpenTelemetry.tracer_provider
+
+        if current_provider.is_a?(OpenTelemetry::SDK::Trace::TracerProvider)
+          # Use existing provider
+          Log.debug("Using existing OpenTelemetry tracer provider")
+          tracer_provider = current_provider
+        else
+          # Create new provider and set as global
+          tracer_provider = OpenTelemetry::SDK::Trace::TracerProvider.new
+          OpenTelemetry.tracer_provider = tracer_provider
+          Log.debug("Created OpenTelemetry tracer provider")
+        end
+      end
+
+      # Enable Braintrust tracing (adds span processor)
+      enable(tracer_provider, state: state)
+    end
+
     def self.enable(tracer_provider, state: nil, exporter: nil)
       state ||= Braintrust.current_state
       raise Error, "No state available" unless state
@@ -94,14 +123,8 @@ module Braintrust
 
       # Build the permalink URL based on parent type
       if parent_type == "experiment_id"
-        # For experiments: {app_url}/app/{org}/p/{project}/experiments/{experiment_id}?r={trace_id}&s={span_id}
-        project_name, experiment_id = parent_id.split("/", 2)
-        unless project_name && experiment_id
-          Log.error("Invalid experiment parent format: #{parent_id}")
-          return ""
-        end
-
-        "#{app_url}/app/#{org_name}/p/#{project_name}/experiments/#{experiment_id}?r=#{trace_id}&s=#{span_id}"
+        # For experiments: {app_url}/app/{org}/object?object_type=experiment&object_id={experiment_id}&r={trace_id}&s={span_id}
+        "#{app_url}/app/#{org_name}/object?object_type=experiment&object_id=#{parent_id}&r=#{trace_id}&s=#{span_id}"
       else
         # For projects: {app_url}/app/{org}/p/{project}/logs?r={trace_id}&s={span_id}
         # parent_type is typically "project_name"

data/lib/braintrust/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Braintrust
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/braintrust.rb

@@ -15,7 +15,7 @@ require_relative "braintrust/eval"
 # @example Initialize with global state
 #   Braintrust.init(
 #     api_key: ENV['BRAINTRUST_API_KEY'],
-#     project: "my-project"
+#     default_project: "my-project"
 #   )
 #
 # @example Initialize with explicit state
@@ -27,42 +27,31 @@ module Braintrust
   class Error < StandardError; end
 
   # Initialize Braintrust SDK
-  # Creates a State from config (ENV + options) and optionally sets it as global
   #
-  # By default, kicks off an async background login that retries indefinitely.
-  # Use blocking_login: true to login synchronously before returning.
-  #
-  # @param set_global [Boolean] whether to set as global state (default: true)
-  # @param blocking_login [Boolean] whether to block and login synchronously (default: false, which starts async login)
-  # @param tracing [Boolean] whether to enable OpenTelemetry tracing (default: true)
-  # @param tracer_provider [TracerProvider, nil] Optional tracer provider to use instead of creating one
   # @param api_key [String, nil] Braintrust API key (overrides BRAINTRUST_API_KEY env var)
   # @param org_name [String, nil] Organization name (overrides BRAINTRUST_ORG_NAME env var)
   # @param default_project [String, nil] Default project for spans (overrides BRAINTRUST_DEFAULT_PROJECT env var, format: "project_name:my-project" or "project_id:uuid")
   # @param app_url [String, nil] App URL (overrides BRAINTRUST_APP_URL env var, default: https://www.braintrust.dev)
   # @param api_url [String, nil] API URL (overrides BRAINTRUST_API_URL env var, default: https://api.braintrust.dev)
+  # @param set_global [Boolean] Whether to set as global state (default: true)
+  # @param blocking_login [Boolean] Whether to block and login synchronously (default: false - async background login)
+  # @param enable_tracing [Boolean] Whether to enable OpenTelemetry tracing (default: true)
+  # @param tracer_provider [TracerProvider, nil] Optional tracer provider to use instead of creating one
   # @return [State] the created state
-  def self.init(set_global: true, blocking_login: false, tracing: true, tracer_provider: nil, **options)
-    config = Config.from_env(**options)
-    state = State.new(
-      api_key: config.api_key,
-      org_name: config.org_name,
-      default_project: config.default_project,
-      app_url: config.app_url,
-      api_url: config.api_url
+  def self.init(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil, set_global: true, blocking_login: false, enable_tracing: true, tracer_provider: nil)
+    state = State.from_env(
+      api_key: api_key,
+      org_name: org_name,
+      default_project: default_project,
+      app_url: app_url,
+      api_url: api_url,
+      blocking_login: blocking_login,
+      enable_tracing: enable_tracing,
+      tracer_provider: tracer_provider
     )
 
     State.global = state if set_global
 
-    # Login: either blocking (synchronous) or async (background thread with retries)
-    if blocking_login
-      state.login
-    else
-      state.login_in_thread  # Default: async background login
-    end
-
-    setup_tracing(state, tracer_provider) if tracing
-
     state
   end
 
@@ -71,40 +60,4 @@ module Braintrust
   def self.current_state
     State.global
   end
-
-  class << self
-    private
-
-    # Set up OpenTelemetry tracing with Braintrust
-    # @param state [State] Braintrust state
-    # @param explicit_provider [TracerProvider, nil] Optional explicit tracer provider
-    # @return [void]
-    def setup_tracing(state, explicit_provider = nil)
-      require "opentelemetry/sdk"
-
-      if explicit_provider
-        # Use the explicitly provided tracer provider
-        # DO NOT set as global - user is managing it themselves
-        Log.debug("Using explicitly provided OpenTelemetry tracer provider")
-        tracer_provider = explicit_provider
-      else
-        # Check if global tracer provider is already a real TracerProvider
-        current_provider = OpenTelemetry.tracer_provider
-
-        if current_provider.is_a?(OpenTelemetry::SDK::Trace::TracerProvider)
-          # Use existing provider
-          Log.debug("Using existing OpenTelemetry tracer provider")
-          tracer_provider = current_provider
-        else
-          # Create new provider and set as global
-          tracer_provider = OpenTelemetry::SDK::Trace::TracerProvider.new
-          OpenTelemetry.tracer_provider = tracer_provider
-          Log.debug("Created OpenTelemetry tracer provider")
-        end
-      end
-
-      # Enable Braintrust tracing (adds span processor)
-      Trace.enable(tracer_provider, state: state)
-    end
-  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: braintrust
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Braintrust
@@ -149,8 +149,35 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.5'
-description: OpenTelemetry-based SDK for Braintrust with tracing, OpenAI integration,
-  and evals
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: kramdown
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: 'Braintrust Ruby SDK for evals, tracing and more. '
 email:
 - info@braintrust.dev
 executables: []
@@ -184,7 +211,7 @@ licenses:
 metadata:
   homepage_uri: https://github.com/braintrustdata/braintrust-sdk-ruby
   source_code_uri: https://github.com/braintrustdata/braintrust-sdk-ruby
-  changelog_uri: https://github.com/braintrustdata/braintrust-sdk-ruby/blob/main/CHANGELOG.md
+  changelog_uri: https://github.com/braintrustdata/braintrust-sdk-ruby/releases
 rdoc_options: []
 require_paths:
 - lib