braintrust 0.1.4 → 0.2.1

data/lib/braintrust/eval/functions.rb

@@ -1,144 +1,22 @@
 # frozen_string_literal: true
 
-require_relative "../api"
-require_relative "scorer"
-require "opentelemetry/sdk"
-require "json"
+require_relative "../functions"
 
 module Braintrust
   module Eval
-    # Functions provides remote function execution capabilities
-    # Allows calling prompts hosted on Braintrust servers as tasks or scorers
+    # @deprecated Use {Braintrust::Functions} instead.
     module Functions
       class << self
-        # Create a task callable that invokes a remote function
-        # @param project [String] Project name
-        # @param slug [String] Function slug
-        # @param state [State, nil] Braintrust state (defaults to global)
-        # @param tracer_provider [TracerProvider, nil] OpenTelemetry tracer provider
-        # @return [Proc] Callable that accepts input and returns output
-        def task(project:, slug:, state: nil, tracer_provider: nil)
-          state ||= Braintrust.current_state
-          raise Error, "No state available" unless state
-
-          # Resolve function ID from project + slug
-          api = API.new(state: state)
-          function_metadata = resolve_function(api, project, slug)
-          function_id = function_metadata["id"]
-          function_name = function_metadata["name"] || slug
-
-          # Get tracer for creating spans
-          tracer_provider ||= OpenTelemetry.tracer_provider
-          tracer = tracer_provider.tracer("braintrust.functions")
-
-          # Return a lambda that invokes the remote function with tracing
-          lambda do |input|
-            # Create a span for the function invocation
-            tracer.in_span("function: #{slug}") do |span|
-              span.set_attribute("braintrust.span_attributes", JSON.dump({type: "function"}))
-              span.set_attribute("braintrust.input_json", JSON.dump(input))
-              span.set_attribute("braintrust.function.name", function_name)
-              span.set_attribute("braintrust.function.id", function_id)
-              span.set_attribute("braintrust.function.slug", slug)
-
-              begin
-                # Invoke the function via API
-                output = api.functions.invoke(id: function_id, input: input)
-                span.set_attribute("braintrust.output_json", JSON.dump(output))
-                output
-              rescue => e
-                # Record exception and set error status
-                span.record_exception(e)
-                span.status = OpenTelemetry::Trace::Status.error(e.message)
-                raise
-              end
-            end
-          end
-        end
-
-        # Create a scorer that invokes a remote function
-        # @param project [String] Project name
-        # @param slug [String] Function slug
-        # @param state [State, nil] Braintrust state (defaults to global)
-        # @param tracer_provider [TracerProvider, nil] OpenTelemetry tracer provider
-        # @return [Scorer] Scorer object that invokes remote function
-        def scorer(project:, slug:, state: nil, tracer_provider: nil)
-          state ||= Braintrust.current_state
-          raise Error, "No state available" unless state
-
-          # Resolve function ID from project + slug
-          api = API.new(state: state)
-          function_metadata = resolve_function(api, project, slug)
-          function_id = function_metadata["id"]
-          function_name = function_metadata["name"] || slug
-
-          # Get tracer for creating spans
-          tracer_provider ||= OpenTelemetry.tracer_provider
-          tracer = tracer_provider.tracer("braintrust.functions")
-
-          # Create a scorer that invokes the remote function
-          Scorer.new(slug) do |input, expected, output, metadata|
-            # Create a span for the function invocation
-            tracer.in_span("function: #{slug}") do |span|
-              scorer_input = {
-                input: input,
-                expected: expected,
-                output: output,
-                metadata: metadata
-              }
-
-              span.set_attribute("braintrust.span_attributes", JSON.dump({type: "function"}))
-              span.set_attribute("braintrust.input_json", JSON.dump(scorer_input))
-              span.set_attribute("braintrust.function.name", function_name)
-              span.set_attribute("braintrust.function.id", function_id)
-              span.set_attribute("braintrust.function.slug", slug)
-
-              begin
-                # Invoke the function via API
-                # The remote scorer receives all scorer arguments
-                result = api.functions.invoke(id: function_id, input: scorer_input)
-
-                score = case result
-                when Hash
-                  if result.key?("score")
-                    result["score"].to_f
-                  else
-                    raise Error, "Hash result must contain 'score' key"
-                  end
-                when String
-                  result.to_f
-                else
-                  raise Error, "Unsupported result type: #{result.class}"
-                end
-
-                span.set_attribute("braintrust.output_json", JSON.dump(score))
-                score
-              rescue => e
-                # Record exception and set error status
-                span.record_exception(e)
-                span.status = OpenTelemetry::Trace::Status.error(e.message)
-                raise
-              end
-            end
-          end
+        # @deprecated Use {Braintrust::Functions.task} instead.
+        def task(**kwargs)
+          Log.warn_once(:eval_functions_task, "Braintrust::Eval::Functions.task is deprecated: use Braintrust::Functions.task instead.")
+          Braintrust::Functions.task(**kwargs)
         end
 
-        private
-
-        # Resolve function ID from project name and slug
-        # @param api [API] API client
-        # @param project [String] Project name
-        # @param slug [String] Function slug
-        # @return [Hash] Function metadata
-        def resolve_function(api, project, slug)
-          result = api.functions.list(project_name: project, slug: slug)
-          functions = result["objects"]
-
-          if functions.nil? || functions.empty?
-            raise Error, "Function '#{slug}' not found in project '#{project}'"
-          end
-
-          functions.first
+        # @deprecated Use {Braintrust::Functions.scorer} instead.
+        def scorer(**kwargs)
+          Log.warn_once(:eval_functions_scorer, "Braintrust::Eval::Functions.scorer is deprecated: use Braintrust::Functions.scorer instead.")
+          Braintrust::Functions.scorer(**kwargs)
         end
       end
     end

data/lib/braintrust/eval/runner.rb

@@ -1,56 +1,52 @@
 # frozen_string_literal: true
 
 require_relative "case"
-require_relative "cases"
-require_relative "scorer"
 require_relative "result"
 require_relative "summary"
+require_relative "trace"
 require_relative "../internal/thread_pool"
+require_relative "../api/internal/btql"
 
 require "opentelemetry/sdk"
 require "json"
 
 module Braintrust
   module Eval
-    # Internal runner class that performs the execution of the Eval and returns the result
+    # Internal runner class that performs the execution of the Eval and returns the result.
+    # Receives a fully-normalized Context — all callables are already typed wrappers.
     class Runner
       # Maximum parallelism allowed (mirrors Internal::ThreadPool::MAX_PARALLELISM)
       MAX_PARALLELISM = Internal::ThreadPool::MAX_PARALLELISM
 
-      def initialize(experiment_id:, experiment_name:, project_id:, project_name:,
-        task:, scorers:, api:, tracer_provider: nil)
-        @experiment_id = experiment_id
-        @experiment_name = experiment_name
-        @project_id = project_id
-        @project_name = project_name
-        @task = task
-        @scorers = normalize_scorers(scorers)
-        @api = api
-        @tracer_provider = tracer_provider || OpenTelemetry.tracer_provider
-        @tracer = @tracer_provider.tracer("braintrust-eval")
-        @parent_attr = "experiment_id:#{experiment_id}"
+      # Per-case mutable accumulator. Built from Case, populated by task and scoring stages.
+      CaseContext = Struct.new(:input, :expected, :output, :metadata, :tags, :trace, :origin, keyword_init: true)
+
+      # @param eval_context [Context] Normalized eval context
+      def initialize(eval_context)
+        @eval_context = eval_context
+        tracer_provider = eval_context.tracer_provider || OpenTelemetry.tracer_provider
+        @tracer = tracer_provider.tracer("braintrust-eval")
 
         # Mutex for thread-safe score collection
         @score_mutex = Mutex.new
       end
 
       # Run evaluation and return Result
-      # @param cases [Array, Enumerable] Test cases
       # @param parallelism [Integer] Number of parallel workers (default: 1)
       # @return [Result]
-      def run(cases, parallelism: 1)
+      def run(parallelism: 1)
         start_time = Time.now
-        normalized_cases = normalize_cases(cases)
+        eval_cases = eval_context.cases
         errors = Queue.new
         @scores = {} # Reset for each run: { scorer_name => Array<Numeric> }
 
         if parallelism && parallelism > 1
-          Internal::ThreadPool.each(normalized_cases, parallelism: parallelism) do |test_case|
-            run_case(test_case, errors)
+          Internal::ThreadPool.each(eval_cases, parallelism: parallelism) do |eval_case|
+            run_eval_case(build_case_context(eval_case), errors)
           end
         else
-          normalized_cases.each do |test_case|
-            run_case(test_case, errors)
+          eval_cases.each do |eval_case|
+            run_eval_case(build_case_context(eval_case), errors)
           end
         end
 
@@ -60,14 +56,16 @@ module Braintrust
         # Calculate duration
         duration = Time.now - start_time
 
-        # Generate permalink
-        permalink = @api.object_permalink(object_type: "experiment", object_id: experiment_id)
+        # Generate permalink (only when state and experiment are available)
+        permalink = if eval_context.state && eval_context.experiment_id
+          eval_context.state.object_permalink(object_type: "experiment", object_id: eval_context.experiment_id)
+        end
 
         Result.new(
-          experiment_id: experiment_id,
-          experiment_name: experiment_name,
-          project_id: project_id,
-          project_name: project_name,
+          experiment_id: eval_context.experiment_id,
+          experiment_name: eval_context.experiment_name,
+          project_id: eval_context.project_id,
+          project_name: eval_context.project_name,
           permalink: permalink,
           errors: error_array,
           duration: duration,
@@ -77,63 +75,71 @@ module Braintrust
 
       private
 
-      attr_reader :experiment_id, :experiment_name, :project_id, :project_name,
-        :task, :scorers, :tracer, :parent_attr
+      attr_reader :eval_context, :tracer
 
       # Run a single test case with OpenTelemetry tracing
       # Creates eval span (parent) with task and score as children
-      # @param test_case [Case] The test case
+      # @param case_context [CaseContext] The per-case accumulator
       # @param errors [Queue] Thread-safe error collection queue
-      def run_case(test_case, errors)
+      def run_eval_case(case_context, errors)
         tracer.in_span("eval") do |eval_span|
-          eval_span.set_attribute("braintrust.parent", parent_attr)
+          eval_span.set_attribute("braintrust.parent", eval_context.parent_span_attr) if eval_context.parent_span_attr
 
           # Set tags early so they're present even if task fails
-          eval_span.set_attribute("braintrust.tags", test_case.tags) if test_case.tags
+          eval_span.set_attribute("braintrust.tags", case_context.tags) if case_context.tags
 
           # Run task
-          output = nil
           begin
-            output = run_task(test_case)
+            case_context.output = run_task(case_context)
           rescue => e
             # Error already recorded on task span, set eval span status
             eval_span.status = OpenTelemetry::Trace::Status.error(e.message)
-            errors << "Task failed for input '#{test_case.input}': #{e.message}"
+            errors << "Task failed for input '#{case_context.input}': #{e.message}"
+            report_progress(eval_span, case_context, error: e.message)
             next
           end
 
+          # Flush spans so they're queryable via BTQL, then build trace
+          eval_context.tracer_provider&.force_flush
+          case_context.trace = build_trace(eval_span)
+
           # Run scorers
+          case_scores = nil
           begin
-            run_scorers(test_case, output)
+            case_scores = run_scorers(case_context)
           rescue => e
             # Error already recorded on score span, set eval span status
             eval_span.status = OpenTelemetry::Trace::Status.error(e.message)
-            errors << "Scorers failed for input '#{test_case.input}': #{e.message}"
+            errors << "Scorers failed for input '#{case_context.input}': #{e.message}"
           end
 
           # Set eval span attributes (after task and scorers complete)
-          set_json_attr(eval_span, "braintrust.span_attributes", {type: "eval"})
-          set_json_attr(eval_span, "braintrust.input_json", test_case.input)
-          set_json_attr(eval_span, "braintrust.output_json", output)
-          set_json_attr(eval_span, "braintrust.expected", test_case.expected) if test_case.expected
+          set_json_attr(eval_span, "braintrust.span_attributes", build_span_attributes("eval"))
+          set_json_attr(eval_span, "braintrust.input_json", case_context.input)
+          set_json_attr(eval_span, "braintrust.output_json", case_context.output)
+          set_json_attr(eval_span, "braintrust.expected", case_context.expected) if case_context.expected
 
           # Set origin for cases from remote sources (already JSON-serialized)
-          eval_span.set_attribute("braintrust.origin", test_case.origin) if test_case.origin
+          eval_span.set_attribute("braintrust.origin", case_context.origin) if case_context.origin
+
+          report_progress(eval_span, case_context, data: case_context.output, scores: case_scores || {})
         end
       end
 
       # Run task with OpenTelemetry tracing
       # Creates task span with input and output
-      # @param test_case [Case] The test case
+      # @param case_context [CaseContext] The per-case context
       # @return [Object] Task output
-      def run_task(test_case)
+      def run_task(case_context)
         tracer.in_span("task") do |task_span|
-          task_span.set_attribute("braintrust.parent", parent_attr)
-          set_json_attr(task_span, "braintrust.span_attributes", {type: "task"})
-          set_json_attr(task_span, "braintrust.input_json", test_case.input)
+          task_span.set_attribute("braintrust.parent", eval_context.parent_span_attr) if eval_context.parent_span_attr
+          set_json_attr(task_span, "braintrust.span_attributes", build_span_attributes("task"))
+          set_json_attr(task_span, "braintrust.input_json", case_context.input)
 
           begin
-            output = task.call(test_case.input)
+            output = eval_context.task.call(
+              input: case_context.input
+            )
             set_json_attr(task_span, "braintrust.output_json", output)
             output
           rescue => e
@@ -147,17 +153,24 @@ module Braintrust
 
       # Run scorers with OpenTelemetry tracing
       # Creates single score span for all scorers
-      # @param test_case [Case] The test case
-      # @param output [Object] Task output
-      def run_scorers(test_case, output)
+      # @param case_context [CaseContext] The per-case context (output must be populated)
+      # @return [Hash] Scores hash { scorer_name => score_value }
+      def run_scorers(case_context)
         tracer.in_span("score") do |score_span|
-          score_span.set_attribute("braintrust.parent", parent_attr)
-          set_json_attr(score_span, "braintrust.span_attributes", {type: "score"})
-
+          score_span.set_attribute("braintrust.parent", eval_context.parent_span_attr) if eval_context.parent_span_attr
+          set_json_attr(score_span, "braintrust.span_attributes", build_span_attributes("score"))
+
+          scorer_kwargs = {
+            input: case_context.input,
+            expected: case_context.expected,
+            output: case_context.output,
+            metadata: case_context.metadata || {},
+            trace: case_context.trace
+          }
           scores = {}
           scorer_error = nil
-          scorers.each do |scorer|
-            score_value = scorer.call(test_case.input, test_case.expected, output, test_case.metadata || {})
+          eval_context.scorers.each do |scorer|
+            score_value = scorer.call(**scorer_kwargs)
             scores[scorer.name] = score_value
 
             # Collect raw score for summary (thread-safe)
@@ -173,39 +186,49 @@ module Braintrust
 
           # Raise after setting scores so we can see which scorers succeeded
           raise scorer_error if scorer_error
+
+          scores
         end
       end
 
-      # Normalize cases input to Cases wrapper
-      # @param cases_input [Array, Enumerable, Cases] The cases input
-      # @return [Cases]
-      def normalize_cases(cases_input)
-        case cases_input
-        when Cases
-          cases_input
-        when Array, Enumerable
-          Cases.new(cases_input)
-        else
-          if cases_input.respond_to?(:each)
-            Cases.new(cases_input)
-          else
-            raise ArgumentError, "cases must be Array or Enumerable"
-          end
-        end
+      # Build a lazy Trace for a case, backed by BTQL.
+      # Returns nil when state or experiment_id are unavailable (local-only mode).
+      # @param eval_span [OpenTelemetry::Trace::Span] The eval span for this case
+      # @return [Eval::Trace, nil]
+      def build_trace(eval_span)
+        return nil unless eval_context.state && eval_context.experiment_id
+
+        root_span_id = eval_span.context.hex_trace_id
+        object_type = "experiment"
+        object_id = eval_context.experiment_id
+        btql = API::Internal::BTQL.new(eval_context.state)
+
+        Eval::Trace.new(
+          spans: -> { btql.trace_spans(object_type: object_type, object_id: object_id, root_span_id: root_span_id) }
+        )
       end
 
-      # Normalize scorers to Scorer objects
-      # @param scorers_input [Array] The scorers input (Scorer objects or callables)
-      # @return [Array<Scorer>]
-      def normalize_scorers(scorers_input)
-        scorers_input.map do |scorer|
-          case scorer
-          when Scorer
-            scorer
-          else
-            Scorer.new(scorer)
-          end
+      # Build a CaseContext from a Case struct
+      # @param eval_case [Case] The eval case
+      # @return [CaseContext]
+      def build_case_context(eval_case)
+        CaseContext.new(
+          input: eval_case.input, expected: eval_case.expected,
+          metadata: eval_case.metadata, tags: eval_case.tags, origin: eval_case.origin
+        )
+      end
+
+      # Report progress for a case via on_progress callback.
+      # Rescues errors in the callback so a broken handler never crashes the eval.
+      def report_progress(eval_span, case_context, **fields)
+        return unless eval_context.on_progress
+        progress = {"id" => eval_span.context.hex_span_id}.merge(fields.transform_keys(&:to_s))
+        if case_context.origin
+          progress["origin"] = case_context.origin.is_a?(String) ? JSON.parse(case_context.origin) : case_context.origin
         end
+        eval_context.on_progress.call(progress)
+      rescue => e
+        Braintrust.logger.warn("on_progress callback error: #{e.message}")
       end
 
       # Record error on span with exception event and error status
@@ -221,6 +244,17 @@ module Braintrust
         span.status = OpenTelemetry::Trace::Status.error(error.message)
       end
 
+      # Build span_attributes hash with type, and optionally name and generation.
+      # Matches Java SDK behavior of including these on every span.
+      # @param type [String] Span type ("eval", "task", or "score")
+      # @return [Hash]
+      def build_span_attributes(type)
+        attrs = {type: type}
+        attrs[:name] = eval_context.experiment_name if eval_context.experiment_name
+        attrs[:generation] = eval_context.generation if eval_context.generation
+        attrs
+      end
+
       # Set a span attribute by JSON encoding the value
       # @param span [OpenTelemetry::Trace::Span] The span
       # @param key [String] The attribute key

data/lib/braintrust/eval/scorer.rb

@@ -1,106 +1,38 @@
 # frozen_string_literal: true
 
+require_relative "../scorer"
+
 module Braintrust
   module Eval
-    # Scorer wraps a scoring function that evaluates task output against expected values
-    # Scorers can accept 3 params (input, expected, output) or 4 params (input, expected, output, metadata)
-    # They can return a float, hash, or array of hashes
-    class Scorer
-      attr_reader :name
-
-      # Create a new scorer
-      # @param name_or_callable [String, Symbol, #call] Name or callable (if callable, name is auto-detected)
-      # @param callable [#call, nil] Callable if name was provided separately
-      # @param block [Proc, nil] Block if no callable provided
-      def initialize(name_or_callable = nil, callable = nil, &block)
-        # Determine name and callable from arguments
-        if name_or_callable.nil? && callable.nil? && block.nil?
-          raise ArgumentError, "Must provide callable or block"
-        end
+    # @deprecated Use {Braintrust::Scorer} instead.
+    module Scorer
+      # @deprecated Use {Braintrust::Scorer.new} instead.
+      def self.new(name_or_callable = nil, callable = nil, &block)
+        Log.warn_once(:eval_scorer_class, "Braintrust::Eval::Scorer is deprecated: use Braintrust::Scorer.new instead.")
 
-        # If first arg is a string/symbol, it's the name
         if name_or_callable.is_a?(String) || name_or_callable.is_a?(Symbol)
-          @name = name_or_callable.to_s
-          @callable = callable || block
-          raise ArgumentError, "Must provide callable or block" unless @callable
+          name = name_or_callable.to_s
+          block = callable.method(:call) if callable && !block
         else
-          # First arg is the callable, try to auto-detect name
-          @callable = name_or_callable || callable || block
-          @name = detect_name(@callable)
-        end
-
-        # Validate callable
-        unless @callable.respond_to?(:call)
-          raise ArgumentError, "Scorer must be callable (respond to :call)"
-        end
-
-        # Detect arity and wrap callable if needed
-        @wrapped_callable = wrap_callable(@callable)
-      end
-
-      # Call the scorer
-      # @param input [Object] The input to the task
-      # @param expected [Object] The expected output
-      # @param output [Object] The actual output from the task
-      # @param metadata [Hash] Optional metadata
-      # @return [Float, Hash, Array] Score value(s)
-      def call(input, expected, output, metadata = {})
-        @wrapped_callable.call(input, expected, output, metadata)
-      end
-
-      private
-
-      # Detect the name from a callable object
-      # @param callable [#call] The callable
-      # @return [String] The detected name
-      def detect_name(callable)
-        # Method objects have .name
-        if callable.is_a?(Method)
-          return callable.name.to_s
+          resolved = name_or_callable || callable
+          block = resolved.method(:call) if resolved && !block
+          name = nil
         end
 
-        # Objects with .name method
-        if callable.respond_to?(:name)
-          return callable.name.to_s
-        end
-
-        # Fallback
-        "scorer"
-      end
-
-      # Wrap the callable to always accept 4 parameters
-      # @param callable [#call] The callable to wrap
-      # @return [Proc] Wrapped callable that accepts 4 params
-      def wrap_callable(callable)
-        arity = callable_arity(callable)
-
-        case arity
-        when 3
-          # Callable takes 3 params - wrap to ignore metadata
-          ->(input, expected, output, metadata) {
-            callable.call(input, expected, output)
-          }
-        when 4, -4, -1
-          # Callable takes 4 params (or variadic with 4+)
-          # -4 means optional 4th param
-          # -1 means variadic (*args)
-          callable
-        else
-          raise ArgumentError, "Scorer must accept 3 or 4 parameters (got arity #{arity})"
-        end
+        scorer = Braintrust::Scorer.new(name, &block)
+        scorer.singleton_class.prepend(PositionalArgsRemapping)
+        scorer
       end
 
-      # Get the arity of a callable
-      # @param callable [#call] The callable
-      # @return [Integer] The arity
-      def callable_arity(callable)
-        if callable.respond_to?(:arity)
-          callable.arity
-        elsif callable.respond_to?(:method)
-          callable.method(:call).arity
-        else
-          # Assume 3 params if we can't detect
-          3
+      # @deprecated Maps positional #call(input, expected, output, metadata) to keyword args.
+      # Will be removed when the legacy Eval::Scorer API is removed.
+      module PositionalArgsRemapping
+        def call(*args, **kwargs)
+          if args.any?
+            Log.warn_once(:scorer_positional_call, "Calling a Scorer with positional args is deprecated: use keyword args (input:, expected:, output:, metadata:) instead.")
+            kwargs = {input: args[0], expected: args[1], output: args[2], metadata: args[3]}
+          end
+          super(**kwargs)
         end
       end
     end