braintrust 0.4.2 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1afc7372056311aec7ccaaae11c5d7a9204cdfc826a34db1bd67126769d253e3
-  data.tar.gz: 6492ea96b8f616c05241983baaeb935cdc3d206ced063781c44981b6d0817e64
+  metadata.gz: '0681cdc4f7c2a0a17a634d61d6137413012c89d0c53683ae730725385d8cd37c'
+  data.tar.gz: d5a143e99f067d34401d5e6eb06fbcdde51839094617428ed14a90600c2f5250
 SHA512:
-  metadata.gz: e6471fc6f3b8beba508dfef0deebfa5f10b1044aaf44817cfb341a71396655ec034b9eca70aaf2e4efb0afa9007a5c284564b3d3471ba1dd18edfd336ea51cb8
-  data.tar.gz: 333ad63d354f902c5e4c5d25d2d67e5ed4f4843fcbfeb7c33a6f273de9dcd7b1c77202417816dd9b834881a3d3a833c68418e81c9a1365ec2e503da3ada027ac
+  metadata.gz: 6fd5bdc6d41fb19d4af576a94bccd02fe0a8212d5e1c0367124bb81fa0e8079849e6eb4a5a4e0e43c170982f11318983fc65a51ff0dfad897835b7e1c40d099b
+  data.tar.gz: 47d6e8ca51a4046549a55acef62c35f09e6e8f303d85d4c7cf706ed13a24dd50a88cd94c40fdcce445251362b4574efa33f646322767e278d11f520fc4ca81a1

data/README.md

@@ -17,9 +17,10 @@ This is the official Ruby SDK for [Braintrust](https://www.braintrust.dev), for
   - [Braintrust.init](#braintrustinit)
   - [Environment variables](#environment-variables)
 - [Tracing](#tracing)
-  - [Supported providers](#supported-providers)
+  - [Supported integrations](#supported-integrations)
   - [Manually applying instrumentation](#manually-applying-instrumentation)
   - [Creating custom spans](#creating-custom-spans)
+  - [Span customizers](#span-customizers)
   - [Attachments](#attachments)
   - [Viewing traces](#viewing-traces)
 - [Evals](#evals)
@@ -120,6 +121,7 @@ Braintrust.init
 | `filter_ai_spans` | `ENV['BRAINTRUST_OTEL_FILTER_AI_SPANS']` | Only export AI-related spans                                                |
 | `org_name`        | `ENV['BRAINTRUST_ORG_NAME']`             | Organization name                                                           |
 | `set_global`      | `true`                                   | Set as global state. Set to `false` for isolated instances                  |
+| `span_customizers` | `[]`                                   | Redact or transform spans before export (see [Span customizers](#span-customizers))                 |
 
 **Example with options:**
 
@@ -148,16 +150,17 @@ Braintrust.init(
 
 ## Tracing
 
-### Supported providers
+### Supported integrations
 
-The SDK automatically instruments these LLM libraries:
+The SDK automatically instruments these libraries:
 
-| Provider  | Gem           | Versions | Integration Name | Examples                                  |
-| --------- | ------------- | -------- | ---------------- | ----------------------------------------- |
-| Anthropic | `anthropic`   | >= 0.3.0 | `:anthropic`     | [Link](./examples/contrib/anthropic.rb)   |
-| OpenAI    | `openai`      | >= 0.1.0 | `:openai`        | [Link](./examples/contrib/openai.rb)      |
-|           | `ruby-openai` | >= 7.0.0 | `:ruby_openai`   | [Link](./examples/contrib/ruby-openai.rb) |
-| Multiple  | `ruby_llm`    | >= 1.8.0 | `:ruby_llm`      | [Link](./examples/contrib/ruby_llm.rb)    |
+| Category | Gem           | Versions | Integration Name | Examples                                        |
+| --------- | ------------- | -------- | ---------------- | ----------------------------------------------- |
+| Anthropic | `anthropic`   | >= 0.3.0 | `:anthropic`     | [Link](./examples/contrib/anthropic/basic.rb)   |
+| OpenAI    | `openai`      | >= 0.1.0 | `:openai`        | [Link](./examples/contrib/openai/basic.rb)      |
+|           | `ruby-openai` | >= 7.0.0 | `:ruby_openai`   | [Link](./examples/contrib/ruby_openai/basic.rb) |
+| Multiple  | `ruby_llm`    | >= 1.8.0 | `:ruby_llm`      | [Link](./examples/contrib/ruby_llm/basic.rb)    |
+| Workflows | `roast-ai`    | >= 1.0.0 | `:roast`         | [Link](./examples/contrib/roast/basic.rb)       |
 
 ### Manually applying instrumentation
 
@@ -192,6 +195,33 @@ tracer.in_span("process-request") do |span|
 end
 ```
 
+### Span customizers
+
+Redact or transform span data before it leaves your application, e.g. to strip PII from LLM inputs and outputs:
+
+```ruby
+# Any object that responds to #on_span_export
+class RedactContent
+  def on_span_export(span)
+    %w[braintrust.input_json braintrust.output_json].each do |key|
+      span.attributes[key] = JSON.generate("[redacted]") if span.attributes.key?(key)
+    end
+    span.attributes.delete("user.email")
+    span
+  end
+end
+
+Braintrust.init(span_customizers: [RedactContent.new])
+```
+
+**When configuring customizers...**
+
+- Do not change the `trace_id`, `span_id`, or `parent_span_id` fields.
+- Always return the span (or a replacement `OpenTelemetry::SDK::Trace::SpanData`).
+- Keep them fast; otherwise they may slow down exporting.
+- Customizers run in order on every finished span sent to Braintrust.
+- If it raises or returns something invalid, the batch is dropped and an error is logged.
+
 ### Attachments
 
 Log binary data (images, PDFs, audio) in your traces:
@@ -567,7 +597,7 @@ The dev server requires the `rack` gem and a Rack-compatible web server.
 | [Passenger](https://www.phusionpassenger.com/) | 6.x               |                                      |
 | [WEBrick](https://github.com/ruby/webrick)     | Not supported     | Does not support server-sent events. |
 
-See examples: [server/eval.ru](./examples/server/eval.ru), 
+See examples: [server/eval.ru](./examples/server/eval.ru),
 
 ## Documentation
 

data/lib/braintrust/config.rb

@@ -8,10 +8,10 @@ module Braintrust
   # and allows overriding with explicit options
   class Config
     attr_reader :api_key, :org_name, :default_project, :app_url, :api_url,
-      :filter_ai_spans, :span_filter_funcs
+      :filter_ai_spans, :span_filter_funcs, :span_customizers
 
     def initialize(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil,
-      filter_ai_spans: nil, span_filter_funcs: nil)
+      filter_ai_spans: nil, span_filter_funcs: nil, span_customizers: nil)
       @api_key = api_key
       @org_name = org_name
       @default_project = default_project
@@ -19,6 +19,7 @@ module Braintrust
       @api_url = api_url
       @filter_ai_spans = filter_ai_spans
       @span_filter_funcs = span_filter_funcs || []
+      @span_customizers = (span_customizers || []).dup.freeze
     end
 
     # Create a Config from environment variables, with option overrides
@@ -30,9 +31,10 @@ module Braintrust
     # @param api_url [String, nil] API URL (overrides BRAINTRUST_API_URL env var)
     # @param filter_ai_spans [Boolean, nil] Enable AI span filtering (overrides BRAINTRUST_OTEL_FILTER_AI_SPANS env var)
     # @param span_filter_funcs [Array<Proc>, nil] Custom span filter functions
+    # @param span_customizers [Array<#on_span_export>, nil] Ordered export customizers (copied and frozen)
     # @return [Config] the created config
     def self.from_env(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil,
-      filter_ai_spans: nil, span_filter_funcs: nil)
+      filter_ai_spans: nil, span_filter_funcs: nil, span_customizers: nil)
       # Parse filter_ai_spans from ENV if not explicitly provided
       env_filter_ai_spans = ENV["BRAINTRUST_OTEL_FILTER_AI_SPANS"]
       filter_ai_spans_value = if filter_ai_spans.nil?
@@ -48,7 +50,8 @@ module Braintrust
         app_url: app_url || ENV["BRAINTRUST_APP_URL"] || "https://www.braintrust.dev",
         api_url: api_url || ENV["BRAINTRUST_API_URL"] || "https://api.braintrust.dev",
         filter_ai_spans: filter_ai_spans_value,
-        span_filter_funcs: span_filter_funcs
+        span_filter_funcs: span_filter_funcs,
+        span_customizers: span_customizers
       )
     end
   end

data/lib/braintrust/contrib/rails/server/templates/initializer.rb.tt

@@ -0,0 +1,19 @@
+require "braintrust/contrib/rails/server"
+
+Braintrust::Contrib::Rails::Server::Engine.configure do |config|
+  config.evaluators = {
+<% if @evaluators.empty? -%>
+    # Add evaluator instances here, for example:
+    # "food-classifier" => FoodClassifier.new
+<% else -%>
+<% @evaluators.each_with_index do |evaluator, index| -%>
+    "<%= evaluator[:slug] %>" => <%= evaluator[:class_name] %>.new<%= "," unless index == @evaluators.length - 1 %>
+<% end -%>
+<% end -%>
+  }
+
+  # Default is :clerk_token. Use :none only when developing locally without
+  # incoming request authentication; outgoing Braintrust API calls still need
+  # a valid Braintrust API key.
+  config.auth = :clerk_token
+end

data/lib/braintrust/contrib/roast/instrumentation/cog.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require_relative "../../support/otel"
+require_relative "../tracing"
+
+module Braintrust
+  module Contrib
+    module Roast
+      module Instrumentation
+        # Wrap the asynchronous work block rather than Cog#run! itself: run! only
+        # schedules a task, while the cog executes later in a different fiber.
+        module Cog
+          class BarrierProxy
+            def initialize(barrier, &around_task)
+              @barrier = barrier
+              @around_task = around_task
+            end
+
+            def async(**options, &task)
+              @barrier.async(**options) do |async_task|
+                @around_task.call(async_task, task)
+              end
+            end
+          end
+
+          def run!(barrier, config, input_context, executor_scope_value, executor_scope_index)
+            parent_context = roast_parent_context
+            return super unless tracing_enabled? && parent_context
+
+            tracer = Braintrust::Contrib::Roast::Tracing.tracer_for(self)
+            proxy = BarrierProxy.new(barrier) do |async_task, task|
+              ::OpenTelemetry::Context.with_current(parent_context) do
+                span_name = "roast.cog.#{name}"
+                tracer.in_span(span_name) do |span|
+                  # Braintrust uses span_attributes for type/name; Roast details go in metadata.
+                  Support::OTel.set_json_attr(span, "braintrust.span_attributes", {type: "task", name: span_name})
+                  Braintrust::Contrib::Context.set!(self, roast_span_context: ::OpenTelemetry::Context.current)
+                  task.call(async_task)
+                rescue => e
+                  span.record_exception(e)
+                  span.status = ::OpenTelemetry::Trace::Status.error(e.message)
+                  raise
+                ensure
+                  record_outcome(span)
+                end
+              end
+            end
+            super(proxy, config, input_context, executor_scope_value, executor_scope_index)
+          end
+
+          private
+
+          def roast_parent_context
+            path = ::Roast::TaskContext.path
+            path.reverse_each do |element|
+              context = Braintrust::Contrib.context_for(element.cog)&.[](:roast_span_context)
+              return context if context
+            end
+            path.reverse_each do |element|
+              context = Braintrust::Contrib.context_for(element.execution_manager&.workflow_context)&.[](:roast_span_context)
+              return context if context
+            end
+            nil
+          end
+
+          def tracing_enabled?
+            context = Braintrust::Contrib.context_for(self)
+            class_context = Braintrust::Contrib.context_for(self.class)
+            context&.[](:enabled) != false && class_context&.[](:enabled) != false
+          end
+
+          def record_outcome(span)
+            outcome = if failed?
+              "failed"
+            elsif skipped?
+              "skipped"
+            else
+              "completed"
+            end
+            Support::OTel.set_json_attr(span, "braintrust.metadata", {
+              "contrib.roast.cog.type" => type,
+              "contrib.roast.cog.name" => name.to_s,
+              "contrib.roast.cog.outcome" => outcome
+            })
+
+            output_value = case output
+            when ::Roast::Cogs::Ruby::Output then output.value
+            when ::Roast::Cogs::Chat::Output then output.response
+            end
+            Support::OTel.set_json_attr(span, "braintrust.output_json", output_value) if output_value.is_a?(String)
+            span.status = ::OpenTelemetry::Trace::Status.error("Cog failed") if failed?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/contrib/roast/instrumentation/workflow.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative "../../support/otel"
+require_relative "../tracing"
+
+module Braintrust
+  module Contrib
+    module Roast
+      module Instrumentation
+        # from_file enters a new Async fiber, so carry its caller's OTel context
+        # through the params object that is handed to the workflow.
+        module WorkflowClass
+          def from_file(workflow_path, params)
+            previous_parent = Braintrust::Contrib.context_for(params)&.[](:roast_parent_context)
+            Braintrust::Contrib::Context.set!(params, roast_parent_context: ::OpenTelemetry::Context.current)
+            super
+          ensure
+            Braintrust::Contrib::Context.set!(params, roast_parent_context: previous_parent)
+          end
+        end
+
+        module Workflow
+          def start!
+            context = Braintrust::Contrib.context_for(self)
+            class_context = Braintrust::Contrib.context_for(self.class)
+            return super if context&.[](:enabled) == false || class_context&.[](:enabled) == false
+
+            tracer = Braintrust::Contrib::Roast::Tracing.tracer_for(self)
+            workflow_context = @workflow_context
+            workflow_path = @workflow_path
+            params = workflow_context.params
+            parent_context = Braintrust::Contrib.context_for(params)&.[](:roast_parent_context)
+            trace_workflow = proc do
+              tracer.in_span("roast.workflow") do |span|
+                Support::OTel.set_json_attr(span, "braintrust.span_attributes", {type: "task", name: "roast.workflow"})
+                Support::OTel.set_json_attr(span, "braintrust.metadata", {"contrib.roast.workflow.name" => File.basename(workflow_path.to_s)})
+                input = {targets: params.targets, args: params.args, kwargs: params.kwargs}.reject { |_, value| value.nil? || value.empty? }
+                Support::OTel.set_json_attr(span, "braintrust.input_json", input) unless input.empty?
+                Braintrust::Contrib::Context.set!(workflow_context, roast_span_context: ::OpenTelemetry::Context.current)
+                super()
+              end
+            end
+            parent_context ? ::OpenTelemetry::Context.with_current(parent_context, &trace_workflow) : trace_workflow.call
+          end
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/contrib/roast/integration.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative "../integration"
+
+module Braintrust
+  module Contrib
+    module Roast
+      # Roast workflow and cog instrumentation.
+      class Integration
+        include Braintrust::Contrib::Integration
+
+        GEM_NAMES = ["roast-ai"].freeze
+        REQUIRE_PATHS = ["roast"].freeze
+        MINIMUM_VERSION = "1.0.0"
+
+        def self.integration_name = :roast
+        def self.gem_names = GEM_NAMES
+        def self.require_paths = REQUIRE_PATHS
+        def self.minimum_version = MINIMUM_VERSION
+
+        def self.loaded?
+          (defined?(::Roast::Workflow) && defined?(::Roast::Cog)) ? true : false
+        end
+
+        def self.patchers
+          require_relative "patcher"
+          [WorkflowPatcher, CogPatcher]
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/contrib/roast/patcher.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "../patcher"
+require_relative "integration"
+require_relative "instrumentation/workflow"
+require_relative "instrumentation/cog"
+
+module Braintrust
+  module Contrib
+    module Roast
+      class WorkflowPatcher < Braintrust::Contrib::Patcher
+        def self.applicable? = defined?(::Roast::Workflow)
+
+        def self.patched?(**options)
+          klass = options[:target]&.singleton_class || ::Roast::Workflow
+          klass.ancestors.include?(Instrumentation::Workflow) &&
+            (options[:target] || ::Roast::Workflow.singleton_class.ancestors.include?(Instrumentation::WorkflowClass))
+        end
+
+        def self.perform_patch(**options)
+          target = options[:target]
+          raise ArgumentError, "target must be a Roast::Workflow" if target && !target.is_a?(::Roast::Workflow)
+
+          (target&.singleton_class || ::Roast::Workflow).prepend(Instrumentation::Workflow)
+          ::Roast::Workflow.singleton_class.prepend(Instrumentation::WorkflowClass) unless target
+        end
+      end
+
+      class CogPatcher < Braintrust::Contrib::Patcher
+        def self.applicable? = defined?(::Roast::Cog)
+
+        def self.patched?(**)
+          ::Roast::Cog.ancestors.include?(Instrumentation::Cog)
+        end
+
+        def self.perform_patch(**)
+          ::Roast::Cog.prepend(Instrumentation::Cog)
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/contrib/roast/tracing.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Contrib
+    module Roast
+      module Tracing
+        INSTRUMENTATION_NAME = "braintrust.contrib.roast"
+
+        def self.tracer_for(target)
+          Braintrust::Contrib.tracer_for(target, name: INSTRUMENTATION_NAME)
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb

@@ -186,17 +186,17 @@ module Braintrust
                     ::RubyLLM::Providers::OpenAI::Tools.tool_for(tool)
                   elsif defined?(::RubyLLM::Providers::Anthropic) && provider.is_a?(::RubyLLM::Providers::Anthropic)
                     ::RubyLLM::Providers::Anthropic::Tools.tool_for(tool)
-                  elsif tool.respond_to?(:params_schema) && tool.params_schema
+                  elsif tool_params_schema(tool)
                     build_basic_tool_schema(tool)
                   else
                     build_minimal_tool_schema(tool)
                   end
                 rescue NameError, ArgumentError => e
                   Braintrust::Log.debug("Failed to extract tool schema using provider-specific method: #{e.class.name}: #{e.message}")
-                  tool_schema = (tool.respond_to?(:params_schema) && tool.params_schema) ? build_basic_tool_schema(tool) : build_minimal_tool_schema(tool)
+                  tool_schema = tool_params_schema(tool) ? build_basic_tool_schema(tool) : build_minimal_tool_schema(tool)
                 end
               else
-                tool_schema = (tool.respond_to?(:params_schema) && tool.params_schema) ? build_basic_tool_schema(tool) : build_minimal_tool_schema(tool)
+                tool_schema = tool_params_schema(tool) ? build_basic_tool_schema(tool) : build_minimal_tool_schema(tool)
               end
 
               # Strip RubyLLM-specific fields to match native OpenAI format
@@ -217,6 +217,19 @@ module Braintrust
               tool_schema
             end
 
+            # A tool's JSON Schema, across ruby_llm versions.
+            # 1.x exposes it as params_schema; 2.0 renamed it to parameters_schema.
+            # @param tool [Object] the RubyLLM tool
+            # @return [Hash, nil] the schema, or nil when the tool declares none
+            def tool_params_schema(tool)
+              %i[params_schema parameters_schema].each do |name|
+                next unless tool.respond_to?(name)
+                schema = tool.public_send(name)
+                return schema if schema
+              end
+              nil
+            end
+
             # Build a basic tool schema with parameters
             def build_basic_tool_schema(tool)
               {
@@ -224,7 +237,7 @@ module Braintrust
                 "function" => {
                   "name" => tool.name.to_s,
                   "description" => tool.description,
-                  "parameters" => tool.params_schema
+                  "parameters" => tool_params_schema(tool)
                 }
               }
             end
@@ -255,10 +268,11 @@ module Braintrust
               # Handle content
               if msg.respond_to?(:content) && msg.content
                 raw_content = msg.content
+                attachments = extract_attachments(msg)
 
-                # Check if content is a Content object with attachments (issue #71)
-                formatted["content"] = if raw_content.respond_to?(:text) && raw_content.respond_to?(:attachments) && raw_content.attachments&.any?
-                  format_multipart_content(raw_content)
+                # Include attachments alongside the text when present (issue #71)
+                formatted["content"] = if attachments.any?
+                  format_multipart_content(content_text(raw_content), attachments)
                 else
                   format_simple_content(raw_content, msg.role.to_s)
                 end
@@ -278,17 +292,43 @@ module Braintrust
               formatted
             end
 
+            # Collect a message's attachments.
+            #
+            # ruby_llm 1.x wraps text and attachments in a Content object hanging off
+            # the message; 2.0 removed Content, leaving content a plain String and
+            # exposing attachments on the message itself. Support both.
+            #
+            # @param msg [Object] the RubyLLM message
+            # @return [Array] the message's attachments, empty when there are none
+            def extract_attachments(msg)
+              content = msg.content if msg.respond_to?(:content)
+              if content.respond_to?(:attachments)
+                from_content = Array(content.attachments)
+                return from_content if from_content.any?
+              end
+
+              msg.respond_to?(:attachments) ? Array(msg.attachments) : []
+            end
+
+            # Extract the plain text of a message's content.
+            # @param raw_content [Object] String, or a 1.x Content object
+            # @return [Object] the text
+            def content_text(raw_content)
+              raw_content.respond_to?(:text) ? raw_content.text : raw_content
+            end
+
             # Format multipart content with text and attachments
-            # @param content_obj [Object] Content object with text and attachments
+            # @param text [Object] the message text
+            # @param attachments [Array] the message's attachments
             # @return [Array<Hash>] array of content parts
-            def format_multipart_content(content_obj)
+            def format_multipart_content(text, attachments)
               content_parts = []
 
               # Add text part
-              content_parts << {"type" => "text", "text" => content_obj.text} if content_obj.text
+              content_parts << {"type" => "text", "text" => text} if text
 
               # Add attachment parts (convert to Braintrust format)
-              content_obj.attachments.each do |attachment|
+              attachments.each do |attachment|
                 content_parts << format_attachment_for_input(attachment)
               end
 

data/lib/braintrust/contrib.rb

@@ -197,9 +197,11 @@ require_relative "contrib/openai/integration"
 require_relative "contrib/ruby_openai/integration"
 require_relative "contrib/ruby_llm/integration"
 require_relative "contrib/anthropic/integration"
+require_relative "contrib/roast/integration"
 
 # Register integrations
 Braintrust::Contrib::OpenAI::Integration.register!
 Braintrust::Contrib::RubyOpenAI::Integration.register!
 Braintrust::Contrib::RubyLLM::Integration.register!
 Braintrust::Contrib::Anthropic::Integration.register!
+Braintrust::Contrib::Roast::Integration.register!

data/lib/braintrust/eval/runner.rb

@@ -27,6 +27,11 @@ module Braintrust
         @eval_context = eval_context
         @tracer = eval_context.tracer_provider.tracer("braintrust-eval")
 
+        # Whether any scorer/classifier can receive `trace:`. Computed once: the
+        # per-case flush that makes traces queryable over BTQL is a real network
+        # wait, so it's only worth paying when something will actually consume it.
+        @needs_trace = (eval_context.scorers + eval_context.classifiers).any? { |c| wants_trace?(c) }
+
         # Mutexes for thread-safe result collection
         @score_mutex = Mutex.new
         @classification_mutex = Mutex.new
@@ -55,6 +60,11 @@ module Braintrust
         # Convert Queue to Array after all threads complete
         error_array = [].tap { |a| a << errors.pop until errors.empty? }
 
+        # Deliver any spans still buffered. Callers that supply their own
+        # tracer_provider get no at_exit hook (see Trace.setup), so without this
+        # the tail of the run would wait on the processor's schedule delay.
+        flush_spans
+
         # Calculate duration
         duration = Time.now - start_time
 
@@ -108,9 +118,11 @@ module Braintrust
             next
           end
 
-          # Flush spans so they're queryable via BTQL, then build trace
-          eval_context.tracer_provider.force_flush if eval_context.tracer_provider.respond_to?(:force_flush)
-          kase.trace = build_trace(eval_span)
+          # Build the trace, then flush spans so they're queryable via BTQL.
+          # Both are skipped unless a scorer/classifier declared `trace:` and the
+          # trace is actually resolvable (build_trace returns nil in local-only mode).
+          kase.trace = build_trace(eval_span) if @needs_trace
+          flush_spans if kase.trace
 
           # Run scorers
           begin
@@ -229,6 +241,23 @@ module Braintrust
         end
       end
 
+      # Whether a callable can receive `trace:`: it either declares the keyword
+      # or accepts arbitrary kwargs and may forward it. Reuses the #call_parameters
+      # introspection Internal::Callable::KeywordFilter already depends on.
+      # @param callable [Scorer, Classifier]
+      # @return [Boolean]
+      def wants_trace?(callable)
+        return true unless callable.respond_to?(:call_parameters)
+
+        callable.call_parameters.any? { |type, name| name == :trace || type == :keyrest }
+      end
+
+      # Force the tracer provider to export buffered spans, if it supports it.
+      def flush_spans
+        provider = eval_context.tracer_provider
+        provider.force_flush if provider.respond_to?(:force_flush)
+      end
+
       # Build a lazy Trace for a case, backed by BTQL.
       # Returns nil when state or experiment_id are unavailable (local-only mode).
       # @param eval_span [OpenTelemetry::Trace::Span] The eval span for this case

data/lib/braintrust/scorer.rb

@@ -152,10 +152,10 @@ module Braintrust
           case block.arity
           when 3
             Log.warn_once(:scorer_positional_3, "Scorer with positional params (input, expected, output) is deprecated. Use keyword args: |input:, expected:, output:| instead.")
-            ->(**kw) { block.call(kw[:input], kw[:expected], kw[:output]) }
+            ->(input: nil, expected: nil, output: nil) { block.call(input, expected, output) }
           when 4, -4, -1
             Log.warn_once(:scorer_positional_4, "Scorer with positional params (input, expected, output, metadata) is deprecated. Use keyword args: |input:, expected:, output:, metadata:| instead.")
-            ->(**kw) { block.call(kw[:input], kw[:expected], kw[:output], kw[:metadata]) }
+            ->(input: nil, expected: nil, output: nil, metadata: nil) { block.call(input, expected, output, metadata) }
           else
             raise ArgumentError, "Scorer must accept keyword args or 3-4 positional params (got arity #{block.arity})"
           end

data/lib/braintrust/state.rb

@@ -24,9 +24,10 @@ module Braintrust
     # @param tracer_provider [TracerProvider, nil] Optional tracer provider to use
     # @param filter_ai_spans [Boolean, nil] Enable AI span filtering
     # @param span_filter_funcs [Array<Proc>, nil] Custom span filter functions
+    # @param span_customizers [Array<#on_span_export>, nil] Ordered synchronous export customizers
     # @param exporter [Exporter, nil] Optional exporter override (for testing)
     # @return [State] the created state
-    def self.from_env(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil, blocking_login: false, enable_tracing: true, tracer_provider: nil, filter_ai_spans: nil, span_filter_funcs: nil, exporter: nil)
+    def self.from_env(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil, blocking_login: false, enable_tracing: true, tracer_provider: nil, filter_ai_spans: nil, span_filter_funcs: nil, span_customizers: nil, exporter: nil)
       require_relative "config"
       config = Config.from_env(
         api_key: api_key,
@@ -35,7 +36,8 @@ module Braintrust
         app_url: app_url,
         api_url: api_url,
         filter_ai_spans: filter_ai_spans,
-        span_filter_funcs: span_filter_funcs
+        span_filter_funcs: span_filter_funcs,
+        span_customizers: span_customizers
       )
       new(
         api_key: config.api_key,

data/lib/braintrust/trace/span_customization.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative "span_customizers"
+require_relative "../logger"
+
+module Braintrust
+  module Trace
+    # Span customizer behavior for exporters.
+    #
+    # Prepend it onto any exporter whose +export(span_data, timeout:)+ it can
+    # +super+ into, and the exporter accepts a +span_customizers:+ option on
+    # construction and runs those customizers on every export. Prepend it
+    # before SpanOrigin, so SpanOrigin runs first and hooks see origin-enriched
+    # data:
+    #
+    #   prepend SpanCustomization
+    #   prepend SpanOrigin
+    #
+    # Customization is fail-closed: if any hook fails, none of the batch is
+    # passed on to the wrapped exporter.
+    module SpanCustomization
+      # Consumes +span_customizers:+ and forwards everything else to the exporter.
+      def initialize(*args, span_customizers: nil, **options, &block)
+        @span_customizers = SpanCustomizers.new(span_customizers)
+        super(*args, **options, &block)
+      end
+
+      def export(span_data, timeout: nil)
+        super(span_customizers.customize(span_data), timeout: timeout)
+      rescue SpanCustomizers::Error => e
+        Log.error("Span customization failed; batch not sent: #{e.message}")
+        OpenTelemetry::SDK::Trace::Export::FAILURE
+      end
+
+      private
+
+      attr_reader :span_customizers
+    end
+  end
+end

data/lib/braintrust/trace/span_customizers.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require_relative "span_export_data"
+
+module Braintrust
+  module Trace
+    # Ordered export-time transformations, independent of origin and transport.
+    class SpanCustomizers
+      # A hook failed. Messages are SDK-generated: they identify the customizer
+      # and the kind of failure, never span payloads or the hook's own exception
+      # message, either of which may contain sensitive data.
+      class Error < StandardError; end
+
+      ID_FIELDS = %i[trace_id span_id parent_span_id].freeze
+      private_constant :ID_FIELDS
+
+      def initialize(customizers = nil)
+        @customizers = (customizers || []).dup.freeze
+        @customizers.each { |customizer| validate!(customizer) }
+      end
+
+      # Transform the whole batch before the exporter groups or serializes it.
+      # Raises Error so the exporter can fail the batch without sending.
+      def customize(span_data)
+        return span_data if @customizers.empty?
+
+        # Hooks may call instrumented clients. Suppress tracing so those spans
+        # cannot re-enter the exporter and customize themselves forever.
+        OpenTelemetry::Common::Utilities.untraced do
+          span_data.map { |span| customize_span(span) }
+        end
+      end
+
+      private
+
+      # Reject objects without the hook up front, so a misspelled method name
+      # fails at registration instead of silently skipping redaction.
+      def validate!(customizer)
+        return if customizer.respond_to?(:on_span_export)
+
+        raise ArgumentError, "#{label(customizer)} must implement on_span_export"
+      end
+
+      def customize_span(span)
+        # Snapshot IDs so in-place mutation through to_span_data cannot slip past validation.
+        ids = identity(span)
+        drops = dropped_counts(span)
+        view = nil
+        @customizers.each do |customizer|
+          view ||= writable_view(span)
+          result = invoke(customizer, view)
+          span = unwrap(result, customizer)
+          raise Error, "#{label(customizer)} changed span IDs; trace_id, span_id and parent_span_id must be preserved" unless identity(span) == ids
+
+          view = (result if result.is_a?(SpanExportData))
+        end
+        restore_counts(span, drops)
+      end
+
+      def invoke(customizer, view)
+        customizer.on_span_export(view)
+      # Rescue broadly: hook errors must fail the batch, not kill the export thread.
+      rescue StandardError, ScriptError, SystemStackError => e
+        raise Error, "#{label(customizer)} raised #{e.class}"
+      end
+
+      def unwrap(result, customizer)
+        span = result.is_a?(SpanExportData) ? result.to_span_data : result
+        return span if span.is_a?(OpenTelemetry::SDK::Trace::SpanData)
+
+        raise Error, "#{label(customizer)} returned #{result.class}; hooks must return the span or a replacement SpanData"
+      end
+
+      def label(customizer)
+        customizer.class.name || customizer.class.inspect
+      end
+
+      def identity(span)
+        ID_FIELDS.map { |field| span.public_send(field).dup }
+      end
+
+      # OTel and replacement spans may carry frozen attributes. Only the hash
+      # needs to be writable; values and other nested objects remain shared.
+      def writable_view(span)
+        span.attributes = span.attributes&.dup || {}
+        SpanExportData.new(span)
+      end
+
+      # Preserve drops caused by SDK limits; entries a customizer deletes are not drops.
+      def dropped_counts(span)
+        {
+          attributes: dropped(span.total_recorded_attributes, span.attributes),
+          events: dropped(span.total_recorded_events, span.events),
+          links: dropped(span.total_recorded_links, span.links)
+        }
+      end
+
+      def restore_counts(span, drops)
+        span.total_recorded_attributes = span.attributes&.size.to_i + drops[:attributes]
+        span.total_recorded_events = span.events&.size.to_i + drops[:events]
+        span.total_recorded_links = span.links&.size.to_i + drops[:links]
+        span
+      end
+
+      def dropped(total, entries)
+        [total.to_i - entries&.size.to_i, 0].max
+      end
+    end
+  end
+end

data/lib/braintrust/trace/span_export_data.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "opentelemetry/sdk"
+
+module Braintrust
+  module Trace
+    # Mutable view of completed span data with read-only identity fields.
+    # Attribute values and other nested objects are shared, not deep-copied.
+    class SpanExportData
+      extend Forwardable
+
+      ID_FIELDS = [:trace_id, :span_id, :parent_span_id].freeze
+      MUTABLE_FIELDS = (OpenTelemetry::SDK::Trace::SpanData.members - ID_FIELDS).freeze
+      private_constant :ID_FIELDS, :MUTABLE_FIELDS
+
+      attr_reader(*ID_FIELDS)
+      def_delegators :@span_data, *MUTABLE_FIELDS, *MUTABLE_FIELDS.map { |field| :"#{field}=" }
+      def_delegators :@span_data, :hex_trace_id, :hex_span_id, :hex_parent_span_id, :instrumentation_library
+
+      def initialize(span_data)
+        @span_data = span_data
+        @trace_id = span_data.trace_id.dup.freeze
+        @span_id = span_data.span_id.dup.freeze
+        @parent_span_id = span_data.parent_span_id.dup.freeze
+      end
+
+      # Access the underlying OpenTelemetry representation, e.g. to build a replacement.
+      # Replacement IDs are validated by the exporter after each hook.
+      def to_span_data
+        @span_data
+      end
+    end
+  end
+end

data/lib/braintrust/trace/span_exporter.rb

@@ -3,17 +3,21 @@
 require "opentelemetry/exporter/otlp"
 require_relative "../state"
 require_relative "span_origin"
+require_relative "span_customization"
 
 module Braintrust
   module Trace
     # Custom OTLP exporter for the Braintrust backend. On export it:
     # - stamps span origin provenance onto each SpanData (via the prepended SpanOrigin behavior)
+    # - runs optional customizers (via the prepended SpanCustomization behavior)
     # - groups spans by braintrust.parent and sets the x-bt-parent header per group,
     #   so the backend routes them to the correct experiment/project
     #
     # Thread safety: BatchSpanProcessor serializes export() calls via its
     # @export_mutex, so @headers mutation here is safe.
     class SpanExporter < OpenTelemetry::Exporter::OTLP::Exporter
+      # The last prepended runs first: origin, then customization, then export below.
+      prepend SpanCustomization
       prepend SpanOrigin
 
       PARENT_ATTR_KEY = SpanProcessor::PARENT_ATTR_KEY

data/lib/braintrust/trace/span_origin.rb

@@ -24,11 +24,15 @@ module Braintrust
       # @param span_data [Array<OpenTelemetry::SDK::Trace::SpanData>]
       # @return [Integer] export result from the wrapped exporter
       def export(span_data, timeout: nil)
+        super(SpanOrigin.enrich_batch(span_data), timeout: timeout)
+      end
+
+      # Enrich a batch of SpanData with span origin provenance.
+      def self.enrich_batch(span_data)
         # Environment is process-global and stable; read it once per batch
         # rather than once per span. It is cheap (ENV reads only).
         environment = Internal::Env.detect_environment
-        enriched = span_data.map { |sd| SpanOrigin.enrich(sd, environment: environment) }
-        super(enriched, timeout: timeout)
+        span_data.map { |sd| enrich(sd, environment: environment) }
       end
 
       # Enrich a single SpanData with span origin provenance.

data/lib/braintrust/trace.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "opentelemetry/sdk"
+require "uri"
 require "opentelemetry/exporter/otlp"
 require_relative "trace/span_processor"
 require_relative "trace/span_exporter"
@@ -88,10 +89,16 @@ module Braintrust
       # Get config from state if available
       config ||= state.respond_to?(:config) ? state.config : nil
 
+      # Customizers only run in SpanExporter; refuse to silently skip them (e.g. redaction).
+      if exporter && config&.span_customizers&.any?
+        raise ArgumentError, "span_customizers are not supported with a custom exporter"
+      end
+
       # Create OTLP HTTP exporter unless override provided
       exporter ||= SpanExporter.new(
         endpoint: "#{state.api_url}/otel/v1/traces",
-        api_key: state.api_key
+        api_key: state.api_key,
+        span_customizers: config&.span_customizers
       )
 
       # Use SimpleSpanProcessor for InMemorySpanExporter (testing), BatchSpanProcessor for production
@@ -185,14 +192,18 @@ module Braintrust
         return ""
       end
 
+      # Encode each path segment so names with spaces or slashes remain one segment.
+      escaped_org_name = URI::DEFAULT_PARSER.escape(org_name, /[^A-Za-z0-9\-._~]/)
+      escaped_parent_id = URI::DEFAULT_PARSER.escape(parent_id, /[^A-Za-z0-9\-._~]/)
+
       # Build the permalink URL based on parent type
       if parent_type == "experiment_id"
         # For experiments: {app_url}/app/{org}/object?object_type=experiment&object_id={experiment_id}&r={trace_id}&s={span_id}
-        "#{app_url}/app/#{org_name}/object?object_type=experiment&object_id=#{parent_id}&r=#{trace_id}&s=#{span_id}"
+        "#{app_url}/app/#{escaped_org_name}/object?object_type=experiment&object_id=#{parent_id}&r=#{trace_id}&s=#{span_id}"
       else
         # For projects: {app_url}/app/{org}/p/{project}/logs?r={trace_id}&s={span_id}
         # parent_type is typically "project_name"
-        "#{app_url}/app/#{org_name}/p/#{parent_id}/logs?r=#{trace_id}&s=#{span_id}"
+        "#{app_url}/app/#{escaped_org_name}/p/#{escaped_parent_id}/logs?r=#{trace_id}&s=#{span_id}"
       end
     rescue => e
       Log.error("Failed to generate permalink: #{e.message}")

data/lib/braintrust/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Braintrust
-  VERSION = "0.4.2"
+  VERSION = "0.6.0"
 end

data/lib/braintrust.rb

@@ -42,6 +42,7 @@ module Braintrust
   # @param tracer_provider [TracerProvider, nil] Optional tracer provider to use instead of creating one
   # @param filter_ai_spans [Boolean, nil] Enable AI span filtering (overrides BRAINTRUST_OTEL_FILTER_AI_SPANS env var)
   # @param span_filter_funcs [Array<Proc>, nil] Custom span filter functions
+  # @param span_customizers [Array<#on_span_export>, nil] Ordered synchronous export customizers
   # @param exporter [Exporter, nil] Optional exporter override (for testing)
   # @param auto_instrument [Boolean, Hash, nil] Auto-instrumentation config:
   #   - nil (default): use BRAINTRUST_AUTO_INSTRUMENT env var, default true if not set
@@ -49,7 +50,7 @@ module Braintrust
   #   - false: explicitly disable
   #   - Hash with :only or :except keys for filtering
   # @return [State] the created state
-  def self.init(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil, set_global: true, blocking_login: false, enable_tracing: true, tracer_provider: nil, filter_ai_spans: nil, span_filter_funcs: nil, exporter: nil, auto_instrument: nil)
+  def self.init(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil, set_global: true, blocking_login: false, enable_tracing: true, tracer_provider: nil, filter_ai_spans: nil, span_filter_funcs: nil, span_customizers: nil, exporter: nil, auto_instrument: nil)
     state = State.from_env(
       api_key: api_key,
       org_name: org_name,
@@ -61,6 +62,7 @@ module Braintrust
       tracer_provider: tracer_provider,
       filter_ai_spans: filter_ai_spans,
       span_filter_funcs: span_filter_funcs,
+      span_customizers: span_customizers,
       exporter: exporter
     )
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: braintrust
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.6.0
 platform: ruby
 authors:
 - Braintrust
@@ -118,7 +118,13 @@ files:
 - lib/braintrust/contrib/rails/server/health_controller.rb
 - lib/braintrust/contrib/rails/server/list_controller.rb
 - lib/braintrust/contrib/rails/server/routes.rb
+- lib/braintrust/contrib/rails/server/templates/initializer.rb.tt
 - lib/braintrust/contrib/registry.rb
+- lib/braintrust/contrib/roast/instrumentation/cog.rb
+- lib/braintrust/contrib/roast/instrumentation/workflow.rb
+- lib/braintrust/contrib/roast/integration.rb
+- lib/braintrust/contrib/roast/patcher.rb
+- lib/braintrust/contrib/roast/tracing.rb
 - lib/braintrust/contrib/ruby_llm/deprecated.rb
 - lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb
 - lib/braintrust/contrib/ruby_llm/instrumentation/common.rb
@@ -180,6 +186,9 @@ files:
 - lib/braintrust/task.rb
 - lib/braintrust/trace.rb
 - lib/braintrust/trace/attachment.rb
+- lib/braintrust/trace/span_customization.rb
+- lib/braintrust/trace/span_customizers.rb
+- lib/braintrust/trace/span_export_data.rb
 - lib/braintrust/trace/span_exporter.rb
 - lib/braintrust/trace/span_filter.rb
 - lib/braintrust/trace/span_origin.rb