braintrust 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1047e872fe3a0acf40117bd9a0fcbf4e5024edbe3c1eb3d3bd1e32f60e5ffb18
-  data.tar.gz: e4c412abf3cf748ff157248feb821724ba4ffb76b67aecd8d1735179572afa93
+  metadata.gz: 38aa89e0aba8b3df517d42a95d016cb09cbfbaeac91d16d4caed00f0779415eb
+  data.tar.gz: fd800de8e09a8ac7ca93342d65abd6c19dcdc5bffe2421dd5cf921e1b6b06645
 SHA512:
-  metadata.gz: 41f183ac2bb20f64bca5c576f0decabdea9b7a4bcb46cbcf0c66292eb34f7e75a3b91516cd5520b4a3e207669eff0fa212afd1f75c1d6b172a44ea27bbc91e4b
-  data.tar.gz: 8a98b9ea710b69ac0b4e22cbdbed2b55bfab0d25db5fda2afc872931ce27a4eb9183433db03be6cedcb295dfca2f1fa43583620d848c55e9fa8b19629ecbfb8e
+  metadata.gz: 82c01479c75cbb51bc79a67af1b91c46af0fdefc2895c05a54837fb38a2bd501a5e462e1cff953fc83e189ad9e1da891e39814077bb0cd03fd66970eccb285a9
+  data.tar.gz: '0757861a2df4962b7cdcff628c5638fe361e33b78ba76417dd3c1087c557fffc7920a857a3916f95bc65e3d6f8bb676afb848a27a6fef28dc987b0d7e237c922'

data/lib/braintrust/contrib/rails/server/templates/initializer.rb.tt

@@ -0,0 +1,19 @@
+require "braintrust/contrib/rails/server"
+
+Braintrust::Contrib::Rails::Server::Engine.configure do |config|
+  config.evaluators = {
+<% if @evaluators.empty? -%>
+    # Add evaluator instances here, for example:
+    # "food-classifier" => FoodClassifier.new
+<% else -%>
+<% @evaluators.each_with_index do |evaluator, index| -%>
+    "<%= evaluator[:slug] %>" => <%= evaluator[:class_name] %>.new<%= "," unless index == @evaluators.length - 1 %>
+<% end -%>
+<% end -%>
+  }
+
+  # Default is :clerk_token. Use :none only when developing locally without
+  # incoming request authentication; outgoing Braintrust API calls still need
+  # a valid Braintrust API key.
+  config.auth = :clerk_token
+end

data/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb

@@ -186,17 +186,17 @@ module Braintrust
                     ::RubyLLM::Providers::OpenAI::Tools.tool_for(tool)
                   elsif defined?(::RubyLLM::Providers::Anthropic) && provider.is_a?(::RubyLLM::Providers::Anthropic)
                     ::RubyLLM::Providers::Anthropic::Tools.tool_for(tool)
-                  elsif tool.respond_to?(:params_schema) && tool.params_schema
+                  elsif tool_params_schema(tool)
                     build_basic_tool_schema(tool)
                   else
                     build_minimal_tool_schema(tool)
                   end
                 rescue NameError, ArgumentError => e
                   Braintrust::Log.debug("Failed to extract tool schema using provider-specific method: #{e.class.name}: #{e.message}")
-                  tool_schema = (tool.respond_to?(:params_schema) && tool.params_schema) ? build_basic_tool_schema(tool) : build_minimal_tool_schema(tool)
+                  tool_schema = tool_params_schema(tool) ? build_basic_tool_schema(tool) : build_minimal_tool_schema(tool)
                 end
               else
-                tool_schema = (tool.respond_to?(:params_schema) && tool.params_schema) ? build_basic_tool_schema(tool) : build_minimal_tool_schema(tool)
+                tool_schema = tool_params_schema(tool) ? build_basic_tool_schema(tool) : build_minimal_tool_schema(tool)
               end
 
               # Strip RubyLLM-specific fields to match native OpenAI format
@@ -217,6 +217,19 @@ module Braintrust
               tool_schema
             end
 
+            # A tool's JSON Schema, across ruby_llm versions.
+            # 1.x exposes it as params_schema; 2.0 renamed it to parameters_schema.
+            # @param tool [Object] the RubyLLM tool
+            # @return [Hash, nil] the schema, or nil when the tool declares none
+            def tool_params_schema(tool)
+              %i[params_schema parameters_schema].each do |name|
+                next unless tool.respond_to?(name)
+                schema = tool.public_send(name)
+                return schema if schema
+              end
+              nil
+            end
+
             # Build a basic tool schema with parameters
             def build_basic_tool_schema(tool)
               {
@@ -224,7 +237,7 @@ module Braintrust
                 "function" => {
                   "name" => tool.name.to_s,
                   "description" => tool.description,
-                  "parameters" => tool.params_schema
+                  "parameters" => tool_params_schema(tool)
                 }
               }
             end
@@ -255,10 +268,11 @@ module Braintrust
               # Handle content
               if msg.respond_to?(:content) && msg.content
                 raw_content = msg.content
+                attachments = extract_attachments(msg)
 
-                # Check if content is a Content object with attachments (issue #71)
-                formatted["content"] = if raw_content.respond_to?(:text) && raw_content.respond_to?(:attachments) && raw_content.attachments&.any?
-                  format_multipart_content(raw_content)
+                # Include attachments alongside the text when present (issue #71)
+                formatted["content"] = if attachments.any?
+                  format_multipart_content(content_text(raw_content), attachments)
                 else
                   format_simple_content(raw_content, msg.role.to_s)
                 end
@@ -278,17 +292,43 @@ module Braintrust
               formatted
             end
 
+            # Collect a message's attachments.
+            #
+            # ruby_llm 1.x wraps text and attachments in a Content object hanging off
+            # the message; 2.0 removed Content, leaving content a plain String and
+            # exposing attachments on the message itself. Support both.
+            #
+            # @param msg [Object] the RubyLLM message
+            # @return [Array] the message's attachments, empty when there are none
+            def extract_attachments(msg)
+              content = msg.content if msg.respond_to?(:content)
+              if content.respond_to?(:attachments)
+                from_content = Array(content.attachments)
+                return from_content if from_content.any?
+              end
+
+              msg.respond_to?(:attachments) ? Array(msg.attachments) : []
+            end
+
+            # Extract the plain text of a message's content.
+            # @param raw_content [Object] String, or a 1.x Content object
+            # @return [Object] the text
+            def content_text(raw_content)
+              raw_content.respond_to?(:text) ? raw_content.text : raw_content
+            end
+
             # Format multipart content with text and attachments
-            # @param content_obj [Object] Content object with text and attachments
+            # @param text [Object] the message text
+            # @param attachments [Array] the message's attachments
             # @return [Array<Hash>] array of content parts
-            def format_multipart_content(content_obj)
+            def format_multipart_content(text, attachments)
               content_parts = []
 
               # Add text part
-              content_parts << {"type" => "text", "text" => content_obj.text} if content_obj.text
+              content_parts << {"type" => "text", "text" => text} if text
 
               # Add attachment parts (convert to Braintrust format)
-              content_obj.attachments.each do |attachment|
+              attachments.each do |attachment|
                 content_parts << format_attachment_for_input(attachment)
               end
 

data/lib/braintrust/dataset.rb

@@ -166,14 +166,14 @@ module Braintrust
       record
     end
 
-    # Build origin JSON for tracing/linking
+    # Build origin pointer for tracing/linking
     # @param raw [Hash] Raw record from API
     # @param dataset_id [String] Dataset ID (fallback if not in record)
-    # @return [String, nil] JSON-serialized origin, or nil if record lacks required fields
+    # @return [Hash, nil] Origin pointer, or nil if record lacks required fields
     def build_origin(raw, dataset_id)
       return nil unless raw["id"] && raw["_xact_id"]
 
-      Internal::Origin.to_json(
+      Internal::Origin.build(
         object_type: "dataset",
         object_id: raw["dataset_id"] || dataset_id,
         id: raw["id"],

data/lib/braintrust/eval/runner.rb

@@ -27,6 +27,11 @@ module Braintrust
         @eval_context = eval_context
         @tracer = eval_context.tracer_provider.tracer("braintrust-eval")
 
+        # Whether any scorer/classifier can receive `trace:`. Computed once: the
+        # per-case flush that makes traces queryable over BTQL is a real network
+        # wait, so it's only worth paying when something will actually consume it.
+        @needs_trace = (eval_context.scorers + eval_context.classifiers).any? { |c| wants_trace?(c) }
+
         # Mutexes for thread-safe result collection
         @score_mutex = Mutex.new
         @classification_mutex = Mutex.new
@@ -55,6 +60,11 @@ module Braintrust
         # Convert Queue to Array after all threads complete
         error_array = [].tap { |a| a << errors.pop until errors.empty? }
 
+        # Deliver any spans still buffered. Callers that supply their own
+        # tracer_provider get no at_exit hook (see Trace.setup), so without this
+        # the tail of the run would wait on the processor's schedule delay.
+        flush_spans
+
         # Calculate duration
         duration = Time.now - start_time
 
@@ -95,7 +105,7 @@ module Braintrust
           set_json_attr(eval_span, "braintrust.expected_json", kase.expected) unless kase.expected.nil?
           set_json_attr(eval_span, "braintrust.metadata", kase.metadata) if kase.metadata
           eval_span.set_attribute("braintrust.tags", kase.tags) if kase.tags
-          eval_span.set_attribute("braintrust.origin", kase.origin) if kase.origin
+          set_json_attr(eval_span, "braintrust.origin", kase.origin) if kase.origin
 
           # Run task
           begin
@@ -108,9 +118,11 @@ module Braintrust
             next
           end
 
-          # Flush spans so they're queryable via BTQL, then build trace
-          eval_context.tracer_provider.force_flush if eval_context.tracer_provider.respond_to?(:force_flush)
-          kase.trace = build_trace(eval_span)
+          # Build the trace, then flush spans so they're queryable via BTQL.
+          # Both are skipped unless a scorer/classifier declared `trace:` and the
+          # trace is actually resolvable (build_trace returns nil in local-only mode).
+          kase.trace = build_trace(eval_span) if @needs_trace
+          flush_spans if kase.trace
 
           # Run scorers
           begin
@@ -229,6 +241,23 @@ module Braintrust
         end
       end
 
+      # Whether a callable can receive `trace:`: it either declares the keyword
+      # or accepts arbitrary kwargs and may forward it. Reuses the #call_parameters
+      # introspection Internal::Callable::KeywordFilter already depends on.
+      # @param callable [Scorer, Classifier]
+      # @return [Boolean]
+      def wants_trace?(callable)
+        return true unless callable.respond_to?(:call_parameters)
+
+        callable.call_parameters.any? { |type, name| name == :trace || type == :keyrest }
+      end
+
+      # Force the tracer provider to export buffered spans, if it supports it.
+      def flush_spans
+        provider = eval_context.tracer_provider
+        provider.force_flush if provider.respond_to?(:force_flush)
+      end
+
       # Build a lazy Trace for a case, backed by BTQL.
       # Returns nil when state or experiment_id are unavailable (local-only mode).
       # @param eval_span [OpenTelemetry::Trace::Span] The eval span for this case
@@ -275,9 +304,7 @@ module Braintrust
       def report_progress(eval_span, kase, **fields)
         return unless eval_context.on_progress
         progress = {"id" => eval_span.context.hex_span_id}.merge(fields.transform_keys(&:to_s))
-        if kase.origin
-          progress["origin"] = kase.origin.is_a?(String) ? JSON.parse(kase.origin) : kase.origin
-        end
+        progress["origin"] = kase.origin if kase.origin
         eval_context.on_progress.call(progress)
       rescue => e
         Braintrust.logger.warn("on_progress callback error: #{e.message}")

data/lib/braintrust/eval.rb

@@ -336,7 +336,7 @@ module Braintrust
         # Use pinned version if available, otherwise compute from max(_xact_id)
         version = dataset_obj.version
         version ||= cases
-          .filter_map { |c| c[:origin] && JSON.parse(c[:origin])["_xact_id"] }
+          .filter_map { |c| c[:origin]&.dig("_xact_id") }
           .max
 
         {cases: cases, dataset_id: dataset_obj.id, dataset_version: version}

data/lib/braintrust/internal/origin.rb

@@ -1,27 +1,27 @@
 # frozen_string_literal: true
 
-require "json"
-
 module Braintrust
   module Internal
-    # Origin provides serialization for source object pointers in Braintrust.
-    # Used internally to link spans back to their source records (e.g., dataset rows).
+    # Origin builds source object pointers, which link spans back to the record
+    # they came from (e.g., a dataset row). Pointers stay Hashes internally and
+    # are serialized once at the boundary that needs them, so that inbound
+    # pointers from the wire and ones we build here have the same shape.
     module Origin
-      # Serialize an origin pointer to JSON
+      # Build an origin pointer
       # @param object_type [String] Type of source object (e.g., "dataset", "playground_logs")
       # @param object_id [String] ID of the source object
       # @param id [String] ID of the specific record within the source
       # @param xact_id [String] Transaction ID
       # @param created [String, nil] Creation timestamp
-      # @return [String] JSON-serialized origin
-      def self.to_json(object_type:, object_id:, id:, xact_id:, created:)
-        JSON.dump({
-          object_type: object_type,
-          object_id: object_id,
-          id: id,
-          _xact_id: xact_id,
-          created: created
-        })
+      # @return [Hash] Origin pointer with string keys, matching the wire format
+      def self.build(object_type:, object_id:, id:, xact_id:, created:)
+        {
+          "object_type" => object_type,
+          "object_id" => object_id,
+          "id" => id,
+          "_xact_id" => xact_id,
+          "created" => created
+        }
       end
     end
   end

data/lib/braintrust/scorer.rb

@@ -152,10 +152,10 @@ module Braintrust
           case block.arity
           when 3
             Log.warn_once(:scorer_positional_3, "Scorer with positional params (input, expected, output) is deprecated. Use keyword args: |input:, expected:, output:| instead.")
-            ->(**kw) { block.call(kw[:input], kw[:expected], kw[:output]) }
+            ->(input: nil, expected: nil, output: nil) { block.call(input, expected, output) }
           when 4, -4, -1
             Log.warn_once(:scorer_positional_4, "Scorer with positional params (input, expected, output, metadata) is deprecated. Use keyword args: |input:, expected:, output:, metadata:| instead.")
-            ->(**kw) { block.call(kw[:input], kw[:expected], kw[:output], kw[:metadata]) }
+            ->(input: nil, expected: nil, output: nil, metadata: nil) { block.call(input, expected, output, metadata) }
           else
             raise ArgumentError, "Scorer must accept keyword args or 3-4 positional params (got arity #{block.arity})"
           end

data/lib/braintrust/server/services/eval_service.rb

@@ -2,12 +2,19 @@
 
 require "json"
 
+require_relative "../../eval/case"
+
 module Braintrust
   module Server
     module Services
       # Framework-agnostic service for running evaluations and streaming SSE results.
       # Must be long-lived (not per-request) to preserve the @state_cache across requests.
       class Eval
+        # Row fields this SDK understands, derived from the Case contract so that
+        # adding a field there carries it through here. Anything else on an
+        # inbound row is a field this version has no code for, and is ignored.
+        CASE_FIELDS = Braintrust::Eval::Case.members.map(&:to_s).freeze
+
         def initialize(evaluators)
           @evaluators = evaluators
           @state_mutex = Mutex.new
@@ -177,9 +184,10 @@ module Braintrust
         # Returns [cases, dataset] where exactly one is non-nil.
         def resolve_data_source(data)
           if data.key?("data")
-            cases = data["data"].map do |d|
-              {input: d["input"], expected: d["expected"]}
-            end
+            # Rows arrive inline from the Playground carrying tags, metadata and
+            # an origin pointer back to the row they came from. Carry them all:
+            # the Playground matches streamed results to its grid by origin.
+            cases = data["data"].map { |row| row.slice(*CASE_FIELDS).transform_keys(&:to_sym) }
             [cases, nil]
           elsif data.key?("dataset_id")
             [nil, Braintrust::Dataset::ID.new(id: data["dataset_id"])]

data/lib/braintrust/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Braintrust
-  VERSION = "0.4.1"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: braintrust
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Braintrust
@@ -118,6 +118,7 @@ files:
 - lib/braintrust/contrib/rails/server/health_controller.rb
 - lib/braintrust/contrib/rails/server/list_controller.rb
 - lib/braintrust/contrib/rails/server/routes.rb
+- lib/braintrust/contrib/rails/server/templates/initializer.rb.tt
 - lib/braintrust/contrib/registry.rb
 - lib/braintrust/contrib/ruby_llm/deprecated.rb
 - lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb