braintrust 0.2.0 → 0.3.0

data/lib/braintrust/eval/runner.rb

@@ -1,59 +1,52 @@
 # frozen_string_literal: true
 
 require_relative "case"
-require_relative "cases"
-require_relative "scorer"
 require_relative "result"
 require_relative "summary"
+require_relative "trace"
 require_relative "../internal/thread_pool"
+require_relative "../api/internal/btql"
 
 require "opentelemetry/sdk"
 require "json"
 
 module Braintrust
   module Eval
-    # Internal runner class that performs the execution of the Eval and returns the result
+    # Internal runner class that performs the execution of the Eval and returns the result.
+    # Receives a fully-normalized Context — all callables are already typed wrappers.
     class Runner
       # Maximum parallelism allowed (mirrors Internal::ThreadPool::MAX_PARALLELISM)
       MAX_PARALLELISM = Internal::ThreadPool::MAX_PARALLELISM
 
-      def initialize(task:, scorers:, experiment_id: nil, experiment_name: nil,
-        project_id: nil, project_name: nil, state: nil, tracer_provider: nil,
-        on_progress: nil, parent: nil)
-        @experiment_id = experiment_id
-        @experiment_name = experiment_name
-        @project_id = project_id
-        @project_name = project_name
-        @task = task
-        @scorers = normalize_scorers(scorers)
-        @state = state
-        @tracer_provider = tracer_provider || OpenTelemetry.tracer_provider
-        @tracer = @tracer_provider.tracer("braintrust-eval")
-        @parent_attr = parent ? "#{parent[:object_type]}:#{parent[:object_id]}" : nil
-        @generation = parent&.dig(:generation)
-        @on_progress = on_progress
+      # Per-case mutable accumulator. Built from Case, populated by task and scoring stages.
+      CaseContext = Struct.new(:input, :expected, :output, :metadata, :tags, :trace, :origin, keyword_init: true)
+
+      # @param eval_context [Context] Normalized eval context
+      def initialize(eval_context)
+        @eval_context = eval_context
+        tracer_provider = eval_context.tracer_provider || OpenTelemetry.tracer_provider
+        @tracer = tracer_provider.tracer("braintrust-eval")
 
         # Mutex for thread-safe score collection
         @score_mutex = Mutex.new
       end
 
       # Run evaluation and return Result
-      # @param cases [Array, Enumerable] Test cases
       # @param parallelism [Integer] Number of parallel workers (default: 1)
       # @return [Result]
-      def run(cases, parallelism: 1)
+      def run(parallelism: 1)
         start_time = Time.now
-        normalized_cases = normalize_cases(cases)
+        eval_cases = eval_context.cases
         errors = Queue.new
         @scores = {} # Reset for each run: { scorer_name => Array<Numeric> }
 
         if parallelism && parallelism > 1
-          Internal::ThreadPool.each(normalized_cases, parallelism: parallelism) do |test_case|
-            run_case(test_case, errors)
+          Internal::ThreadPool.each(eval_cases, parallelism: parallelism) do |eval_case|
+            run_eval_case(build_case_context(eval_case), errors)
           end
         else
-          normalized_cases.each do |test_case|
-            run_case(test_case, errors)
+          eval_cases.each do |eval_case|
+            run_eval_case(build_case_context(eval_case), errors)
           end
         end
 
@@ -64,15 +57,15 @@ module Braintrust
         duration = Time.now - start_time
 
         # Generate permalink (only when state and experiment are available)
-        permalink = if @state && experiment_id
-          @state.object_permalink(object_type: "experiment", object_id: experiment_id)
+        permalink = if eval_context.state && eval_context.experiment_id
+          eval_context.state.object_permalink(object_type: "experiment", object_id: eval_context.experiment_id)
         end
 
         Result.new(
-          experiment_id: experiment_id,
-          experiment_name: experiment_name,
-          project_id: project_id,
-          project_name: project_name,
+          experiment_id: eval_context.experiment_id,
+          experiment_name: eval_context.experiment_name,
+          project_id: eval_context.project_id,
+          project_name: eval_context.project_name,
           permalink: permalink,
           errors: error_array,
           duration: duration,
@@ -82,86 +75,73 @@ module Braintrust
 
       private
 
-      attr_reader :experiment_id, :experiment_name, :project_id, :project_name,
-        :task, :scorers, :tracer, :parent_attr
+      attr_reader :eval_context, :tracer
 
       # Run a single test case with OpenTelemetry tracing
       # Creates eval span (parent) with task and score as children
-      # @param test_case [Case] The test case
+      # @param case_context [CaseContext] The per-case accumulator
       # @param errors [Queue] Thread-safe error collection queue
-      def run_case(test_case, errors)
-        tracer.in_span("eval") do |eval_span|
-          eval_span.set_attribute("braintrust.parent", parent_attr) if parent_attr
-
-          # Set tags early so they're present even if task fails
-          eval_span.set_attribute("braintrust.tags", test_case.tags) if test_case.tags
+      def run_eval_case(case_context, errors)
+        # Each eval case starts its own trace — detach from any ambient span context
+        eval_span = tracer.start_root_span("eval")
+        OpenTelemetry::Trace.with_span(eval_span) do
+          # Set attributes known before task execution
+          eval_span.set_attribute("braintrust.parent", eval_context.parent_span_attr) if eval_context.parent_span_attr
+          set_json_attr(eval_span, "braintrust.span_attributes", build_span_attributes("eval"))
+          set_json_attr(eval_span, "braintrust.input_json", {input: case_context.input})
+          set_json_attr(eval_span, "braintrust.expected", case_context.expected) if case_context.expected
+          set_json_attr(eval_span, "braintrust.metadata", case_context.metadata) if case_context.metadata
+          eval_span.set_attribute("braintrust.tags", case_context.tags) if case_context.tags
+          eval_span.set_attribute("braintrust.origin", case_context.origin) if case_context.origin
 
           # Run task
-          output = nil
           begin
-            output = run_task(test_case)
+            case_context.output = run_task(case_context)
           rescue => e
             # Error already recorded on task span, set eval span status
             eval_span.status = OpenTelemetry::Trace::Status.error(e.message)
-            errors << "Task failed for input '#{test_case.input}': #{e.message}"
-            if @on_progress
-              error_progress = {
-                "id" => eval_span.context.hex_span_id,
-                "error" => e.message
-              }
-              if test_case.origin
-                error_progress["origin"] = test_case.origin.is_a?(String) ? JSON.parse(test_case.origin) : test_case.origin
-              end
-              @on_progress.call(error_progress)
-            end
+            set_json_attr(eval_span, "braintrust.output_json", {output: nil})
+            errors << "Task failed for input '#{case_context.input}': #{e.message}"
+            report_progress(eval_span, case_context, error: e.message)
             next
           end
 
+          # Flush spans so they're queryable via BTQL, then build trace
+          eval_context.tracer_provider&.force_flush
+          case_context.trace = build_trace(eval_span)
+
           # Run scorers
-          case_scores = nil
           begin
-            case_scores = run_scorers(test_case, output)
+            run_scorers(case_context)
           rescue => e
             # Error already recorded on score span, set eval span status
             eval_span.status = OpenTelemetry::Trace::Status.error(e.message)
-            errors << "Scorers failed for input '#{test_case.input}': #{e.message}"
+            errors << "Scorers failed for input '#{case_context.input}': #{e.message}"
           end
 
-          # Set eval span attributes (after task and scorers complete)
-          set_json_attr(eval_span, "braintrust.span_attributes", build_span_attributes("eval"))
-          set_json_attr(eval_span, "braintrust.input_json", test_case.input)
-          set_json_attr(eval_span, "braintrust.output_json", output)
-          set_json_attr(eval_span, "braintrust.expected", test_case.expected) if test_case.expected
-
-          # Set origin for cases from remote sources (already JSON-serialized)
-          eval_span.set_attribute("braintrust.origin", test_case.origin) if test_case.origin
-
-          if @on_progress
-            progress = {
-              "id" => eval_span.context.hex_span_id,
-              "data" => output,
-              "scores" => case_scores || {}
-            }
-            if test_case.origin
-              progress["origin"] = test_case.origin.is_a?(String) ? JSON.parse(test_case.origin) : test_case.origin
-            end
-            @on_progress.call(progress)
-          end
+          # Set output after task completes
+          set_json_attr(eval_span, "braintrust.output_json", {output: case_context.output})
+
+          report_progress(eval_span, case_context, data: case_context.output)
         end
+      ensure
+        eval_span&.finish
       end
 
       # Run task with OpenTelemetry tracing
       # Creates task span with input and output
-      # @param test_case [Case] The test case
+      # @param case_context [CaseContext] The per-case context
       # @return [Object] Task output
-      def run_task(test_case)
+      def run_task(case_context)
         tracer.in_span("task") do |task_span|
-          task_span.set_attribute("braintrust.parent", parent_attr) if parent_attr
+          task_span.set_attribute("braintrust.parent", eval_context.parent_span_attr) if eval_context.parent_span_attr
           set_json_attr(task_span, "braintrust.span_attributes", build_span_attributes("task"))
-          set_json_attr(task_span, "braintrust.input_json", test_case.input)
+          set_json_attr(task_span, "braintrust.input_json", case_context.input)
 
           begin
-            output = task.call(test_case.input)
+            output = eval_context.task.call(
+              input: case_context.input
+            )
             set_json_attr(task_span, "braintrust.output_json", output)
             output
           rescue => e
@@ -173,70 +153,103 @@ module Braintrust
         end
       end
 
-      # Run scorers with OpenTelemetry tracing
-      # Creates single score span for all scorers
-      # @param test_case [Case] The test case
-      # @param output [Object] Task output
-      # @return [Hash] Scores hash { scorer_name => score_value }
-      def run_scorers(test_case, output)
-        tracer.in_span("score") do |score_span|
-          score_span.set_attribute("braintrust.parent", parent_attr) if parent_attr
-          set_json_attr(score_span, "braintrust.span_attributes", build_span_attributes("score"))
-
-          scores = {}
-          scorer_error = nil
-          scorers.each do |scorer|
-            score_value = scorer.call(test_case.input, test_case.expected, output, test_case.metadata || {})
-            scores[scorer.name] = score_value
-
-            # Collect raw score for summary (thread-safe)
-            collect_score(scorer.name, score_value)
-          rescue => e
-            # Record first error but continue processing other scorers
-            scorer_error ||= e
-            record_span_error(score_span, e, "ScorerError")
-          end
+      # Run scorers with OpenTelemetry tracing.
+      # Creates one span per scorer, each a direct child of the current (eval) span.
+      # @param case_context [CaseContext] The per-case context (output must be populated)
+      def run_scorers(case_context)
+        scorer_kwargs = {
+          input: case_context.input,
+          expected: case_context.expected,
+          output: case_context.output,
+          metadata: case_context.metadata || {},
+          trace: case_context.trace
+        }
+        scorer_input = {
+          input: case_context.input,
+          expected: case_context.expected,
+          output: case_context.output,
+          metadata: case_context.metadata || {}
+        }
+
+        scorer_error = nil
+        eval_context.scorers.each do |scorer|
+          collect_scores(run_scorer(scorer, scorer_kwargs, scorer_input))
+        rescue => e
+          scorer_error ||= e
+        end
 
-          # Always set scores attribute, even if some scorers failed
-          set_json_attr(score_span, "braintrust.scores", scores)
+        raise scorer_error if scorer_error
+      end
 
-          # Raise after setting scores so we can see which scorers succeeded
-          raise scorer_error if scorer_error
+      # Run a single scorer inside its own span.
+      # @param scorer [Scorer] The scorer to run
+      # @param scorer_kwargs [Hash] Keyword arguments for the scorer
+      # @param scorer_input [Hash] Input to log on the span
+      # @return [Array<Hash>] Raw score results from the scorer
+      def run_scorer(scorer, scorer_kwargs, scorer_input)
+        tracer.in_span(scorer.name) do |score_span|
+          score_span.set_attribute("braintrust.parent", eval_context.parent_span_attr) if eval_context.parent_span_attr
+          set_json_attr(score_span, "braintrust.span_attributes", build_scorer_span_attributes(scorer.name))
+          set_json_attr(score_span, "braintrust.input_json", scorer_input)
+
+          score_results = scorer.call(**scorer_kwargs)
+
+          scorer_scores = {}
+          scorer_metadata = {}
+          score_results.each do |s|
+            scorer_scores[s[:name]] = s[:score]
+            scorer_metadata[s[:name]] = s[:metadata] if s[:metadata].is_a?(Hash)
+          end
+
+          set_json_attr(score_span, "braintrust.output_json", scorer_scores)
+          set_json_attr(score_span, "braintrust.scores", scorer_scores)
+          set_json_attr(score_span, "braintrust.metadata", scorer_metadata) unless scorer_metadata.empty?
 
-          scores
+          score_results
+        rescue => e
+          record_span_error(score_span, e, "ScorerError")
+          raise
         end
       end
 
-      # Normalize cases input to Cases wrapper
-      # @param cases_input [Array, Enumerable, Cases] The cases input
-      # @return [Cases]
-      def normalize_cases(cases_input)
-        case cases_input
-        when Cases
-          cases_input
-        when Array, Enumerable
-          Cases.new(cases_input)
-        else
-          if cases_input.respond_to?(:each)
-            Cases.new(cases_input)
-          else
-            raise ArgumentError, "cases must be Array or Enumerable"
-          end
-        end
+      # Build a lazy Trace for a case, backed by BTQL.
+      # Returns nil when state or experiment_id are unavailable (local-only mode).
+      # @param eval_span [OpenTelemetry::Trace::Span] The eval span for this case
+      # @return [Eval::Trace, nil]
+      def build_trace(eval_span)
+        return nil unless eval_context.state && eval_context.experiment_id
+
+        root_span_id = eval_span.context.hex_trace_id
+        object_type = "experiment"
+        object_id = eval_context.experiment_id
+        btql = API::Internal::BTQL.new(eval_context.state)
+
+        Eval::Trace.new(
+          spans: -> { btql.trace_spans(object_type: object_type, object_id: object_id, root_span_id: root_span_id) }
+        )
       end
 
-      # Normalize scorers to Scorer objects
-      # @param scorers_input [Array] The scorers input (Scorer objects or callables)
-      # @return [Array<Scorer>]
-      def normalize_scorers(scorers_input)
-        scorers_input.map do |scorer|
-          case scorer
-          when Scorer
-            scorer
-          else
-            Scorer.new(scorer)
-          end
+      # Build a CaseContext from a Case struct
+      # @param eval_case [Case] The eval case
+      # @return [CaseContext]
+      def build_case_context(eval_case)
+        CaseContext.new(
+          input: eval_case.input, expected: eval_case.expected,
+          metadata: eval_case.metadata, tags: eval_case.tags, origin: eval_case.origin
+        )
+      end
+
+      # Report progress for a case via on_progress callback.
+      # Rescues errors in the callback so a broken handler never crashes the eval.
+      def report_progress(eval_span, case_context, **fields)
+        return unless eval_context.on_progress
+        progress = {"id" => eval_span.context.hex_span_id}.merge(fields.transform_keys(&:to_s))
+        if case_context.origin
+          progress["origin"] = case_context.origin.is_a?(String) ? JSON.parse(case_context.origin) : case_context.origin
         end
+        eval_context.on_progress.call(progress)
+      rescue => e
+        Braintrust.logger.warn("on_progress callback error: #{e.message}")
       end
 
       # Record error on span with exception event and error status
@@ -258,8 +271,18 @@ module Braintrust
       # @return [Hash]
       def build_span_attributes(type)
         attrs = {type: type}
-        attrs[:name] = experiment_name if experiment_name
-        attrs[:generation] = @generation if @generation
+        attrs[:name] = eval_context.experiment_name if eval_context.experiment_name
+        attrs[:generation] = eval_context.generation if eval_context.generation
+        attrs
+      end
+
+      # Build span_attributes for a scorer span.
+      # Each scorer gets its own span with type "score", purpose "scorer", and the scorer's name.
+      # @param scorer_name [String] The scorer name
+      # @return [Hash]
+      def build_scorer_span_attributes(scorer_name)
+        attrs = {type: "score", name: scorer_name, purpose: "scorer"}
+        attrs[:generation] = eval_context.generation if eval_context.generation
         attrs
       end
 
@@ -271,14 +294,11 @@ module Braintrust
         span.set_attribute(key, JSON.dump(value))
       end
 
-      # Collect a single score value for summary calculation
-      # @param name [String] Scorer name
-      # @param value [Object] Score value (only Numeric values are collected)
-      def collect_score(name, value)
-        return unless value.is_a?(Numeric)
-
+      # Collect score results into the summary accumulator (thread-safe).
+      # @param score_results [Array<Hash>] Score results from a scorer
+      def collect_scores(score_results)
         @score_mutex.synchronize do
-          (@scores[name] ||= []) << value
+          score_results.each { |s| (@scores[s[:name]] ||= []) << s[:score] }
         end
       end
     end

data/lib/braintrust/eval/scorer.rb

@@ -1,112 +1,40 @@
 # frozen_string_literal: true
 
+require_relative "../scorer"
+
 module Braintrust
   module Eval
-    # Scorer wraps a scoring function that evaluates task output against expected values
-    # Scorers can accept 3 params (input, expected, output) or 4 params (input, expected, output, metadata)
-    # They can return a float, hash, or array of hashes
-    class Scorer
-      attr_reader :name
-
-      # Create a new scorer
-      # @param name_or_callable [String, Symbol, #call] Name or callable (if callable, name is auto-detected)
-      # @param callable [#call, nil] Callable if name was provided separately
-      # @param block [Proc, nil] Block if no callable provided
-      def initialize(name_or_callable = nil, callable = nil, &block)
-        # Determine name and callable from arguments
-        if name_or_callable.nil? && callable.nil? && block.nil?
-          raise ArgumentError, "Must provide callable or block"
-        end
+    # @deprecated Use {Braintrust::Scorer} instead.
+    module Scorer
+      # @deprecated Use {Braintrust::Scorer.new} instead.
+      def self.new(name_or_callable = nil, callable = nil, &block)
+        Log.warn_once(:eval_scorer_class, "Braintrust::Eval::Scorer is deprecated: use Braintrust::Scorer.new instead.")
 
-        # If first arg is a string/symbol, it's the name
         if name_or_callable.is_a?(String) || name_or_callable.is_a?(Symbol)
-          @name = name_or_callable.to_s
-          @callable = callable || block
-          raise ArgumentError, "Must provide callable or block" unless @callable
+          name = name_or_callable.to_s
+          block = callable.method(:call) if callable && !block
         else
-          # First arg is the callable, try to auto-detect name
-          @callable = name_or_callable || callable || block
-          @name = detect_name(@callable)
+          resolved = name_or_callable || callable
+          block = resolved.method(:call) if resolved && !block
+          name = nil
         end
 
-        # Validate callable
-        unless @callable.respond_to?(:call)
-          raise ArgumentError, "Scorer must be callable (respond to :call)"
-        end
-
-        # Detect arity and wrap callable if needed
-        @wrapped_callable = wrap_callable(@callable)
+        scorer = Braintrust::Scorer.new(name, &block)
+        scorer.singleton_class.prepend(PositionalArgsRemapping)
+        scorer
       end
 
-      # Call the scorer
-      # @param input [Object] The input to the task
-      # @param expected [Object] The expected output
-      # @param output [Object] The actual output from the task
-      # @param metadata [Hash] Optional metadata
-      # @return [Float, Hash, Array] Score value(s)
-      def call(input, expected, output, metadata = {})
-        @wrapped_callable.call(input, expected, output, metadata)
-      end
-
-      private
-
-      # Detect the name from a callable object
-      # @param callable [#call] The callable
-      # @return [String] The detected name
-      def detect_name(callable)
-        # Method objects have .name
-        if callable.is_a?(Method)
-          return callable.name.to_s
-        end
-
-        # Objects with .name method
-        if callable.respond_to?(:name)
-          return callable.name.to_s
-        end
-
-        # Fallback
-        "scorer"
-      end
-
-      # Wrap the callable to always accept 4 parameters
-      # @param callable [#call] The callable to wrap
-      # @return [Proc] Wrapped callable that accepts 4 params
-      def wrap_callable(callable)
-        arity = callable_arity(callable)
-
-        case arity
-        when 3
-          # Callable takes 3 params - wrap to ignore metadata
-          ->(input, expected, output, metadata) {
-            callable.call(input, expected, output)
-          }
-        when 4, -4, -1
-          # Callable takes 4 params (or variadic with 4+)
-          # -4 means optional 4th param
-          # -1 means variadic (*args)
-          callable
-        else
-          raise ArgumentError, "Scorer must accept 3 or 4 parameters (got arity #{arity})"
-        end
-      end
-
-      # Get the arity of a callable
-      # @param callable [#call] The callable
-      # @return [Integer] The arity
-      def callable_arity(callable)
-        if callable.respond_to?(:arity)
-          callable.arity
-        elsif callable.respond_to?(:method)
-          callable.method(:call).arity
-        else
-          # Assume 3 params if we can't detect
-          3
+      # @deprecated Maps positional #call(input, expected, output, metadata) to keyword args.
+      # Will be removed when the legacy Eval::Scorer API is removed.
+      module PositionalArgsRemapping
+        def call(*args, **kwargs)
+          if args.any?
+            Log.warn_once(:scorer_positional_call, "Calling a Scorer with positional args is deprecated: use keyword args (input:, expected:, output:, metadata:) instead.")
+            kwargs = {input: args[0], expected: args[1], output: args[2], metadata: args[3]}
+          end
+          super(**kwargs)
         end
       end
     end
   end
-
-  # Value object wrapping a remote scorer function UUID.
-  # Used by Eval.run to distinguish remote scorers from local callables.
-  ScorerId = Struct.new(:function_id, :version, keyword_init: true)
 end