braintrust 0.4.2 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1afc7372056311aec7ccaaae11c5d7a9204cdfc826a34db1bd67126769d253e3
-  data.tar.gz: 6492ea96b8f616c05241983baaeb935cdc3d206ced063781c44981b6d0817e64
+  metadata.gz: 38aa89e0aba8b3df517d42a95d016cb09cbfbaeac91d16d4caed00f0779415eb
+  data.tar.gz: fd800de8e09a8ac7ca93342d65abd6c19dcdc5bffe2421dd5cf921e1b6b06645
 SHA512:
-  metadata.gz: e6471fc6f3b8beba508dfef0deebfa5f10b1044aaf44817cfb341a71396655ec034b9eca70aaf2e4efb0afa9007a5c284564b3d3471ba1dd18edfd336ea51cb8
-  data.tar.gz: 333ad63d354f902c5e4c5d25d2d67e5ed4f4843fcbfeb7c33a6f273de9dcd7b1c77202417816dd9b834881a3d3a833c68418e81c9a1365ec2e503da3ada027ac
+  metadata.gz: 82c01479c75cbb51bc79a67af1b91c46af0fdefc2895c05a54837fb38a2bd501a5e462e1cff953fc83e189ad9e1da891e39814077bb0cd03fd66970eccb285a9
+  data.tar.gz: '0757861a2df4962b7cdcff628c5638fe361e33b78ba76417dd3c1087c557fffc7920a857a3916f95bc65e3d6f8bb676afb848a27a6fef28dc987b0d7e237c922'

data/lib/braintrust/contrib/rails/server/templates/initializer.rb.tt

@@ -0,0 +1,19 @@
+require "braintrust/contrib/rails/server"
+
+Braintrust::Contrib::Rails::Server::Engine.configure do |config|
+  config.evaluators = {
+<% if @evaluators.empty? -%>
+    # Add evaluator instances here, for example:
+    # "food-classifier" => FoodClassifier.new
+<% else -%>
+<% @evaluators.each_with_index do |evaluator, index| -%>
+    "<%= evaluator[:slug] %>" => <%= evaluator[:class_name] %>.new<%= "," unless index == @evaluators.length - 1 %>
+<% end -%>
+<% end -%>
+  }
+
+  # Default is :clerk_token. Use :none only when developing locally without
+  # incoming request authentication; outgoing Braintrust API calls still need
+  # a valid Braintrust API key.
+  config.auth = :clerk_token
+end

data/lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb

@@ -186,17 +186,17 @@ module Braintrust
                     ::RubyLLM::Providers::OpenAI::Tools.tool_for(tool)
                   elsif defined?(::RubyLLM::Providers::Anthropic) && provider.is_a?(::RubyLLM::Providers::Anthropic)
                     ::RubyLLM::Providers::Anthropic::Tools.tool_for(tool)
-                  elsif tool.respond_to?(:params_schema) && tool.params_schema
+                  elsif tool_params_schema(tool)
                     build_basic_tool_schema(tool)
                   else
                     build_minimal_tool_schema(tool)
                   end
                 rescue NameError, ArgumentError => e
                   Braintrust::Log.debug("Failed to extract tool schema using provider-specific method: #{e.class.name}: #{e.message}")
-                  tool_schema = (tool.respond_to?(:params_schema) && tool.params_schema) ? build_basic_tool_schema(tool) : build_minimal_tool_schema(tool)
+                  tool_schema = tool_params_schema(tool) ? build_basic_tool_schema(tool) : build_minimal_tool_schema(tool)
                 end
               else
-                tool_schema = (tool.respond_to?(:params_schema) && tool.params_schema) ? build_basic_tool_schema(tool) : build_minimal_tool_schema(tool)
+                tool_schema = tool_params_schema(tool) ? build_basic_tool_schema(tool) : build_minimal_tool_schema(tool)
               end
 
               # Strip RubyLLM-specific fields to match native OpenAI format
@@ -217,6 +217,19 @@ module Braintrust
               tool_schema
             end
 
+            # A tool's JSON Schema, across ruby_llm versions.
+            # 1.x exposes it as params_schema; 2.0 renamed it to parameters_schema.
+            # @param tool [Object] the RubyLLM tool
+            # @return [Hash, nil] the schema, or nil when the tool declares none
+            def tool_params_schema(tool)
+              %i[params_schema parameters_schema].each do |name|
+                next unless tool.respond_to?(name)
+                schema = tool.public_send(name)
+                return schema if schema
+              end
+              nil
+            end
+
             # Build a basic tool schema with parameters
             def build_basic_tool_schema(tool)
               {
@@ -224,7 +237,7 @@ module Braintrust
                 "function" => {
                   "name" => tool.name.to_s,
                   "description" => tool.description,
-                  "parameters" => tool.params_schema
+                  "parameters" => tool_params_schema(tool)
                 }
               }
             end
@@ -255,10 +268,11 @@ module Braintrust
               # Handle content
               if msg.respond_to?(:content) && msg.content
                 raw_content = msg.content
+                attachments = extract_attachments(msg)
 
-                # Check if content is a Content object with attachments (issue #71)
-                formatted["content"] = if raw_content.respond_to?(:text) && raw_content.respond_to?(:attachments) && raw_content.attachments&.any?
-                  format_multipart_content(raw_content)
+                # Include attachments alongside the text when present (issue #71)
+                formatted["content"] = if attachments.any?
+                  format_multipart_content(content_text(raw_content), attachments)
                 else
                   format_simple_content(raw_content, msg.role.to_s)
                 end
@@ -278,17 +292,43 @@ module Braintrust
               formatted
             end
 
+            # Collect a message's attachments.
+            #
+            # ruby_llm 1.x wraps text and attachments in a Content object hanging off
+            # the message; 2.0 removed Content, leaving content a plain String and
+            # exposing attachments on the message itself. Support both.
+            #
+            # @param msg [Object] the RubyLLM message
+            # @return [Array] the message's attachments, empty when there are none
+            def extract_attachments(msg)
+              content = msg.content if msg.respond_to?(:content)
+              if content.respond_to?(:attachments)
+                from_content = Array(content.attachments)
+                return from_content if from_content.any?
+              end
+
+              msg.respond_to?(:attachments) ? Array(msg.attachments) : []
+            end
+
+            # Extract the plain text of a message's content.
+            # @param raw_content [Object] String, or a 1.x Content object
+            # @return [Object] the text
+            def content_text(raw_content)
+              raw_content.respond_to?(:text) ? raw_content.text : raw_content
+            end
+
             # Format multipart content with text and attachments
-            # @param content_obj [Object] Content object with text and attachments
+            # @param text [Object] the message text
+            # @param attachments [Array] the message's attachments
             # @return [Array<Hash>] array of content parts
-            def format_multipart_content(content_obj)
+            def format_multipart_content(text, attachments)
               content_parts = []
 
               # Add text part
-              content_parts << {"type" => "text", "text" => content_obj.text} if content_obj.text
+              content_parts << {"type" => "text", "text" => text} if text
 
               # Add attachment parts (convert to Braintrust format)
-              content_obj.attachments.each do |attachment|
+              attachments.each do |attachment|
                 content_parts << format_attachment_for_input(attachment)
               end
 

data/lib/braintrust/eval/runner.rb

@@ -27,6 +27,11 @@ module Braintrust
         @eval_context = eval_context
         @tracer = eval_context.tracer_provider.tracer("braintrust-eval")
 
+        # Whether any scorer/classifier can receive `trace:`. Computed once: the
+        # per-case flush that makes traces queryable over BTQL is a real network
+        # wait, so it's only worth paying when something will actually consume it.
+        @needs_trace = (eval_context.scorers + eval_context.classifiers).any? { |c| wants_trace?(c) }
+
         # Mutexes for thread-safe result collection
         @score_mutex = Mutex.new
         @classification_mutex = Mutex.new
@@ -55,6 +60,11 @@ module Braintrust
         # Convert Queue to Array after all threads complete
         error_array = [].tap { |a| a << errors.pop until errors.empty? }
 
+        # Deliver any spans still buffered. Callers that supply their own
+        # tracer_provider get no at_exit hook (see Trace.setup), so without this
+        # the tail of the run would wait on the processor's schedule delay.
+        flush_spans
+
         # Calculate duration
         duration = Time.now - start_time
 
@@ -108,9 +118,11 @@ module Braintrust
             next
           end
 
-          # Flush spans so they're queryable via BTQL, then build trace
-          eval_context.tracer_provider.force_flush if eval_context.tracer_provider.respond_to?(:force_flush)
-          kase.trace = build_trace(eval_span)
+          # Build the trace, then flush spans so they're queryable via BTQL.
+          # Both are skipped unless a scorer/classifier declared `trace:` and the
+          # trace is actually resolvable (build_trace returns nil in local-only mode).
+          kase.trace = build_trace(eval_span) if @needs_trace
+          flush_spans if kase.trace
 
           # Run scorers
           begin
@@ -229,6 +241,23 @@ module Braintrust
         end
       end
 
+      # Whether a callable can receive `trace:`: it either declares the keyword
+      # or accepts arbitrary kwargs and may forward it. Reuses the #call_parameters
+      # introspection Internal::Callable::KeywordFilter already depends on.
+      # @param callable [Scorer, Classifier]
+      # @return [Boolean]
+      def wants_trace?(callable)
+        return true unless callable.respond_to?(:call_parameters)
+
+        callable.call_parameters.any? { |type, name| name == :trace || type == :keyrest }
+      end
+
+      # Force the tracer provider to export buffered spans, if it supports it.
+      def flush_spans
+        provider = eval_context.tracer_provider
+        provider.force_flush if provider.respond_to?(:force_flush)
+      end
+
       # Build a lazy Trace for a case, backed by BTQL.
       # Returns nil when state or experiment_id are unavailable (local-only mode).
       # @param eval_span [OpenTelemetry::Trace::Span] The eval span for this case

data/lib/braintrust/scorer.rb

@@ -152,10 +152,10 @@ module Braintrust
           case block.arity
           when 3
             Log.warn_once(:scorer_positional_3, "Scorer with positional params (input, expected, output) is deprecated. Use keyword args: |input:, expected:, output:| instead.")
-            ->(**kw) { block.call(kw[:input], kw[:expected], kw[:output]) }
+            ->(input: nil, expected: nil, output: nil) { block.call(input, expected, output) }
           when 4, -4, -1
             Log.warn_once(:scorer_positional_4, "Scorer with positional params (input, expected, output, metadata) is deprecated. Use keyword args: |input:, expected:, output:, metadata:| instead.")
-            ->(**kw) { block.call(kw[:input], kw[:expected], kw[:output], kw[:metadata]) }
+            ->(input: nil, expected: nil, output: nil, metadata: nil) { block.call(input, expected, output, metadata) }
           else
             raise ArgumentError, "Scorer must accept keyword args or 3-4 positional params (got arity #{block.arity})"
           end

data/lib/braintrust/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Braintrust
-  VERSION = "0.4.2"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: braintrust
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.5.0
 platform: ruby
 authors:
 - Braintrust
@@ -118,6 +118,7 @@ files:
 - lib/braintrust/contrib/rails/server/health_controller.rb
 - lib/braintrust/contrib/rails/server/list_controller.rb
 - lib/braintrust/contrib/rails/server/routes.rb
+- lib/braintrust/contrib/rails/server/templates/initializer.rb.tt
 - lib/braintrust/contrib/registry.rb
 - lib/braintrust/contrib/ruby_llm/deprecated.rb
 - lib/braintrust/contrib/ruby_llm/instrumentation/chat.rb