braintrust 0.2.0 → 0.3.0

data/lib/braintrust/contrib/rails/server.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+begin
+  require "action_controller"
+  require "rails/engine"
+rescue LoadError
+  raise LoadError,
+    "Rails (actionpack + railties) is required for the Braintrust Rails server engine. " \
+    "Add `gem 'rails'` or `gem 'actionpack'` and `gem 'railties'` to your Gemfile."
+end
+
+require "json"
+require_relative "../../eval"
+require_relative "../../server/sse"
+require_relative "../../server/auth/no_auth"
+require_relative "../../server/auth/clerk_token"
+require_relative "../../server/middleware/cors"
+require_relative "../../server/services/list_service"
+require_relative "../../server/services/eval_service"
+require_relative "server/engine"

data/lib/braintrust/dataset.rb

@@ -181,9 +181,12 @@ module Braintrust
         created: raw["created"]
       )
     end
+
+    # Value object wrapping a dataset UUID for resolution by ID.
+    # Used by Eval.run to distinguish dataset-by-ID from dataset-by-name.
+    ID = Struct.new(:id, keyword_init: true)
   end
 
-  # Value object wrapping a dataset UUID for resolution by ID.
-  # Used by Eval.run to distinguish dataset-by-ID from dataset-by-name.
-  DatasetId = Struct.new(:id, keyword_init: true)
+  # @deprecated Use {Braintrust::Dataset::ID} instead.
+  DatasetId = Dataset::ID
 end

data/lib/braintrust/eval/context.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require_relative "cases"
+
+module Braintrust
+  module Eval
+    # Holds all normalized, ready-to-execute eval components.
+    # Use Context.build to construct from raw user inputs.
+    class Context
+      attr_reader :task, :scorers, :cases, :experiment_id, :experiment_name,
+        :project_id, :project_name, :state, :tracer_provider,
+        :on_progress, :parent_span_attr, :generation
+
+      def initialize(task:, scorers:, cases:, experiment_id: nil, experiment_name: nil,
+        project_id: nil, project_name: nil, state: nil, tracer_provider: nil,
+        on_progress: nil, parent_span_attr: nil, generation: nil)
+        @task = task
+        @scorers = scorers
+        @cases = cases
+        @experiment_id = experiment_id
+        @experiment_name = experiment_name
+        @project_id = project_id
+        @project_name = project_name
+        @state = state
+        @tracer_provider = tracer_provider
+        @on_progress = on_progress
+        @parent_span_attr = parent_span_attr
+        @generation = generation
+      end
+
+      # Build a Context from raw user inputs.
+      # Factory normalizes task, scorers, and cases into typed wrappers.
+      # Parent is resolved into parent_span_attr and generation.
+      def self.build(task:, scorers:, cases:, experiment_id: nil, experiment_name: nil,
+        project_id: nil, project_name: nil, state: nil, tracer_provider: nil,
+        on_progress: nil, parent: nil)
+        factory = Factory.new(state: state, tracer_provider: tracer_provider, project_name: project_name)
+
+        Context.new(
+          task: factory.normalize_task(task),
+          scorers: factory.normalize_scorers(scorers),
+          cases: factory.normalize_cases(cases),
+          experiment_id: experiment_id,
+          experiment_name: experiment_name,
+          project_id: project_id,
+          project_name: project_name,
+          state: state,
+          tracer_provider: tracer_provider,
+          on_progress: on_progress,
+          parent_span_attr: factory.resolve_parent_span_attr(parent),
+          generation: parent&.dig(:generation)
+        )
+      end
+
+      # Encapsulates normalization of raw user inputs into typed wrappers.
+      class Factory
+        def initialize(state: nil, tracer_provider: nil, project_name: nil)
+          @state = state
+          @tracer_provider = tracer_provider
+          @project_name = project_name
+        end
+
+        def normalize_cases(raw)
+          case raw
+          when Cases
+            raw
+          when Array, Enumerable
+            Cases.new(raw)
+          else
+            if raw.respond_to?(:each)
+              Cases.new(raw)
+            else
+              raise ArgumentError, "cases must be Array or Enumerable"
+            end
+          end
+        end
+
+        def resolve_parent_span_attr(parent)
+          return nil unless parent
+          "#{parent[:object_type]}:#{parent[:object_id]}"
+        end
+
+        def normalize_task(raw)
+          case raw
+          when Task
+            raw
+          when Proc
+            # Pass Proc/Lambda directly to preserve keyword arg info.
+            # Legacy positional lambdas (arity 1) are auto-wrapped by Task#wrap_block.
+            Task.new(&raw)
+          else
+            # Callable class: wrap via method(:call) to preserve keyword arg info
+            name = raw.respond_to?(:name) ? raw.name : nil
+            Task.new(name, &raw.method(:call))
+          end
+        end
+
+        def normalize_scorers(raw)
+          raw.map do |scorer|
+            case scorer
+            when String
+              raise ArgumentError, "project is required to resolve scorer slug '#{scorer}'" unless @project_name
+              Braintrust::Functions.scorer(
+                project: @project_name,
+                slug: scorer,
+                state: @state,
+                tracer_provider: @tracer_provider
+              )
+            when Braintrust::Scorer::ID
+              Braintrust::Functions.scorer(
+                id: scorer.function_id,
+                version: scorer.version,
+                state: @state,
+                tracer_provider: @tracer_provider
+              )
+            when Braintrust::Scorer
+              scorer
+            when Proc
+              # Pass Proc/Lambda directly to preserve keyword arg info
+              # (method(:call) loses parameter metadata)
+              Braintrust::Scorer.new(&scorer)
+            else
+              name = scorer.respond_to?(:name) ? scorer.name : nil
+              Braintrust::Scorer.new(name, &scorer.method(:call))
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/eval/evaluator.rb

@@ -5,21 +5,27 @@ module Braintrust
     # Base class for evaluators. Subclass and override #task and #scorers,
     # or instantiate directly with keyword arguments.
     #
+    # Evaluators are used with the dev server, which reports scorer names
+    # to the Braintrust UI. Always use named scorers (via Scorer.new or
+    # subclass) so they display meaningfully.
+    #
     # @example Subclass pattern
     #   class FoodClassifier < Braintrust::Eval::Evaluator
     #     def task
-    #       ->(input) { classify(input) }
+    #       ->(input:) { classify(input) }
     #     end
     #
     #     def scorers
-    #       [Braintrust::Eval.scorer("exact_match") { |i, e, o| o == e ? 1.0 : 0.0 }]
+    #       [Braintrust::Scorer.new("exact_match") { |expected:, output:| output == expected ? 1.0 : 0.0 }]
     #     end
     #   end
     #
     # @example Inline pattern
     #   Braintrust::Eval::Evaluator.new(
-    #     task: ->(input) { input.upcase },
-    #     scorers: [my_scorer]
+    #     task: ->(input:) { input.upcase },
+    #     scorers: [
+    #       Braintrust::Scorer.new("exact_match") { |expected:, output:| output == expected ? 1.0 : 0.0 }
+    #     ]
     #   )
     class Evaluator
       attr_accessor :task, :scorers, :parameters
@@ -48,7 +54,7 @@ module Braintrust
       # @param project [String, nil] Project name
       # @param experiment [String, nil] Experiment name
       # @param project_id [String, nil] Project UUID (skips project creation)
-      # @param dataset [String, Hash, Dataset, DatasetId, nil] Dataset to fetch
+      # @param dataset [String, Hash, Dataset, Dataset::ID, nil] Dataset to fetch
       # @param scorers [Array, nil] Additional scorers (merged with evaluator's own)
       # @param parent [Hash, nil] Parent span context
       # @param state [State, nil] Braintrust state

data/lib/braintrust/eval/functions.rb

@@ -1,178 +1,22 @@
 # frozen_string_literal: true
 
-require_relative "../api"
-require_relative "scorer"
-require "opentelemetry/sdk"
-require "json"
+require_relative "../functions"
 
 module Braintrust
   module Eval
-    # Functions provides remote function execution capabilities
-    # Allows calling prompts hosted on Braintrust servers as tasks or scorers
+    # @deprecated Use {Braintrust::Functions} instead.
     module Functions
       class << self
-        # Create a task callable that invokes a remote function
-        # @param project [String] Project name
-        # @param slug [String] Function slug
-        # @param state [State, nil] Braintrust state (defaults to global)
-        # @param tracer_provider [TracerProvider, nil] OpenTelemetry tracer provider
-        # @return [Proc] Callable that accepts input and returns output
-        def task(project:, slug:, state: nil, tracer_provider: nil)
-          state ||= Braintrust.current_state
-          raise Error, "No state available" unless state
-
-          # Resolve function ID from project + slug
-          api = API.new(state: state)
-          function_metadata = resolve_function(api, project, slug)
-          function_id = function_metadata["id"]
-          function_name = function_metadata["name"] || slug
-
-          # Get tracer for creating spans
-          tracer_provider ||= OpenTelemetry.tracer_provider
-          tracer = tracer_provider.tracer("braintrust.functions")
-
-          # Return a lambda that invokes the remote function with tracing
-          lambda do |input|
-            # Create a span for the function invocation
-            tracer.in_span("function: #{slug}") do |span|
-              span.set_attribute("braintrust.span_attributes", JSON.dump({type: "function"}))
-              span.set_attribute("braintrust.input_json", JSON.dump(input))
-              span.set_attribute("braintrust.function.name", function_name)
-              span.set_attribute("braintrust.function.id", function_id)
-              span.set_attribute("braintrust.function.slug", slug)
-
-              begin
-                # Invoke the function via API
-                output = api.functions.invoke(id: function_id, input: input)
-                span.set_attribute("braintrust.output_json", JSON.dump(output))
-                output
-              rescue => e
-                # Record exception and set error status
-                span.record_exception(e)
-                span.status = OpenTelemetry::Trace::Status.error(e.message)
-                raise
-              end
-            end
-          end
-        end
-
-        # Create a scorer that invokes a remote function by ID
-        # @param id [String] Function UUID
-        # @param version [String, nil] Optional version to pin to
-        # @param state [State, nil] Braintrust state (defaults to global)
-        # @param tracer_provider [TracerProvider, nil] OpenTelemetry tracer provider
-        # @return [Scorer] Scorer object that invokes remote function
-        def scorer_by_id(id:, state: nil, version: nil, tracer_provider: nil)
-          state ||= Braintrust.current_state
-          api = API.new(state: state)
-          api.login
-
-          function_metadata = api.functions.get(id: id, version: version)
-          function_id = function_metadata["id"]
-          function_name = function_metadata["name"] || id
-
-          tracer_provider ||= OpenTelemetry.tracer_provider
-          tracer = tracer_provider.tracer("braintrust.functions")
-
-          build_scorer(function_id: function_id, function_name: function_name, api: api, tracer: tracer)
+        # @deprecated Use {Braintrust::Functions.task} instead.
+        def task(**kwargs)
+          Log.warn_once(:eval_functions_task, "Braintrust::Eval::Functions.task is deprecated: use Braintrust::Functions.task instead.")
+          Braintrust::Functions.task(**kwargs)
         end
 
-        # Create a scorer that invokes a remote function
-        # @param project [String] Project name
-        # @param slug [String] Function slug
-        # @param state [State, nil] Braintrust state (defaults to global)
-        # @param tracer_provider [TracerProvider, nil] OpenTelemetry tracer provider
-        # @return [Scorer] Scorer object that invokes remote function
-        def scorer(project:, slug:, state: nil, tracer_provider: nil)
-          state ||= Braintrust.current_state
-          raise Error, "No state available" unless state
-
-          # Resolve function ID from project + slug
-          api = API.new(state: state)
-          function_metadata = resolve_function(api, project, slug)
-          function_id = function_metadata["id"]
-          function_name = function_metadata["name"] || slug
-
-          # Get tracer for creating spans
-          tracer_provider ||= OpenTelemetry.tracer_provider
-          tracer = tracer_provider.tracer("braintrust.functions")
-
-          build_scorer(function_id: function_id, function_name: function_name, api: api, tracer: tracer)
-        end
-
-        private
-
-        # Build a Scorer that invokes a remote function
-        # Shared implementation used by both scorer and scorer_by_id
-        # @param function_id [String] Function UUID
-        # @param function_name [String] Function display name
-        # @param api [API] Braintrust API client
-        # @param tracer [OpenTelemetry::Trace::Tracer] Tracer instance
-        # @return [Scorer]
-        def build_scorer(function_id:, function_name:, api:, tracer:)
-          Scorer.new(function_name) do |input, expected, output, metadata|
-            tracer.in_span("function: #{function_name}") do |span|
-              scorer_input = {
-                input: input,
-                expected: expected,
-                output: output,
-                metadata: metadata
-              }
-
-              span.set_attribute("braintrust.span_attributes", JSON.dump({type: "function"}))
-              span.set_attribute("braintrust.input_json", JSON.dump(scorer_input))
-              span.set_attribute("braintrust.function.name", function_name)
-              span.set_attribute("braintrust.function.id", function_id)
-
-              begin
-                result = api.functions.invoke(id: function_id, input: scorer_input)
-
-                score = case result
-                when Numeric
-                  result.to_f
-                when true
-                  1.0
-                when false
-                  0.0
-                when Hash
-                  if result.key?("score")
-                    result["score"].to_f
-                  else
-                    raise Error, "Hash result must contain 'score' key"
-                  end
-                when String
-                  result.to_f
-                when nil
-                  nil
-                else
-                  raise Error, "Unsupported result type: #{result.class}"
-                end
-
-                span.set_attribute("braintrust.output_json", JSON.dump(score))
-                score
-              rescue => e
-                span.record_exception(e)
-                span.status = OpenTelemetry::Trace::Status.error(e.message)
-                raise
-              end
-            end
-          end
-        end
-
-        # Resolve function ID from project name and slug
-        # @param api [API] API client
-        # @param project [String] Project name
-        # @param slug [String] Function slug
-        # @return [Hash] Function metadata
-        def resolve_function(api, project, slug)
-          result = api.functions.list(project_name: project, slug: slug)
-          functions = result["objects"]
-
-          if functions.nil? || functions.empty?
-            raise Error, "Function '#{slug}' not found in project '#{project}'"
-          end
-
-          functions.first
+        # @deprecated Use {Braintrust::Functions.scorer} instead.
+        def scorer(**kwargs)
+          Log.warn_once(:eval_functions_scorer, "Braintrust::Eval::Functions.scorer is deprecated: use Braintrust::Functions.scorer instead.")
+          Braintrust::Functions.scorer(**kwargs)
         end
       end
     end