braintrust 0.2.1 → 0.3.1

data/lib/braintrust/scorer.rb

@@ -40,12 +40,52 @@ module Braintrust
       Block.new(name: name || DEFAULT_NAME, &block)
     end
 
-    # Included into classes that +include Scorer+. Prepends KeywordFilter
-    # so #call receives only its declared kwargs, and provides a default #name.
+    # Included into classes that +include Scorer+. Prepends KeywordFilter and
+    # ResultNormalizer so #call receives only declared kwargs and always returns
+    # Array<Hash>. Also provides a default #name and #call_parameters.
     module Callable
+      # Normalizes the raw return value of #call into Array<Hash>.
+      # Nested inside Callable because it depends on #name which Callable provides.
+      module ResultNormalizer
+        # @return [Array<Hash>] normalized score hashes with :score, :metadata, :name keys
+        def call(**kwargs)
+          normalize_score_result(super)
+        end
+
+        private
+
+        # @param result [Numeric, Hash, Array<Hash>] raw return value from #call
+        # @return [Array<Hash>] one or more score hashes with :score, :metadata, :name keys
+        # @raise [ArgumentError] if any score value is not Numeric
+        def normalize_score_result(result)
+          case result
+          when Array then result.map { |item| normalize_score_item(item) }
+          when Hash then [normalize_score_item(result)]
+          else
+            raise ArgumentError, "#{name}: score must be Numeric, got #{result.inspect}" unless result.is_a?(Numeric)
+            [{score: result, metadata: nil, name: name}]
+          end
+        end
+
+        # Fills in missing :name from the scorer and validates :score.
+        # @param item [Hash] a score hash with at least a :score key
+        # @return [Hash] the same hash with :name set
+        # @raise [ArgumentError] if :score is not Numeric
+        def normalize_score_item(item)
+          item[:name] ||= name
+          raise ArgumentError, "#{item[:name]}: score must be Numeric, got #{item[:score].inspect}" unless item[:score].is_a?(Numeric)
+          item
+        end
+      end
+
+      # Infrastructure modules prepended onto every scorer class.
+      # Used both to set up the ancestor chain and to skip past them in
+      # #call_parameters so KeywordFilter sees the real call signature.
+      PREPENDED = [Internal::Callable::KeywordFilter, ResultNormalizer].freeze
+
       # @param base [Class] the class including Callable
       def self.included(base)
-        base.prepend(Internal::Callable::KeywordFilter)
+        PREPENDED.each { |mod| base.prepend(mod) }
       end
 
       # Default name derived from the class name (e.g. FuzzyMatch -> "fuzzy_match").
@@ -55,6 +95,17 @@ module Braintrust
         return Scorer::DEFAULT_NAME unless klass
         klass.gsub(/([a-z])([A-Z])/, '\1_\2').downcase
       end
+
+      # Provides KeywordFilter with the actual call signature of the subclass.
+      # Walks past PREPENDED modules in the ancestor chain so that user-defined
+      # #call keyword params are correctly introspected.
+      # Block overrides this to point directly at @block.parameters.
+      # @return [Array<Array>] parameter list
+      def call_parameters
+        meth = method(:call)
+        meth = meth.super_method while meth.super_method && PREPENDED.include?(meth.owner)
+        meth.parameters
+      end
     end
 
     # Block-based scorer. Stores a Proc and delegates #call to it.
@@ -75,7 +126,7 @@ module Braintrust
       end
 
       # @param kwargs [Hash] keyword arguments (filtered by KeywordFilter)
-      # @return [Float, Hash, Array] score result
+      # @return [Array<Hash>] normalized score results
       def call(**kwargs)
         @block.call(**kwargs)
       end

data/lib/braintrust/server/handlers/eval.rb

@@ -10,38 +10,15 @@ module Braintrust
       class Eval
         def initialize(evaluators)
           @evaluators = evaluators
+          @service = Services::Eval.new(evaluators)
         end
 
         def call(env)
           body = parse_body(env)
           return error_response(400, "Invalid JSON body") unless body
 
-          name = body["name"]
-          return error_response(400, "Missing required field: name") unless name
-
-          evaluator = @evaluators[name]
-          return error_response(404, "Evaluator '#{name}' not found") unless evaluator
-
-          data = body["data"]
-          return error_response(400, "Missing required field: data") unless data
-
-          # Validate exactly one data source
-          data_sources = ["data", "dataset_name", "dataset_id"].count { |k| data.key?(k) }
-          return error_response(400, "Exactly one data source required") if data_sources != 1
-
-          experiment_name = body["experiment_name"]
-
-          # Resolve data source
-          cases, dataset = resolve_data_source(data)
-
-          # Resolve remote scorers from request
-          remote_scorer_ids = resolve_remote_scorers(body["scores"])
-
-          # Resolve parent span context
-          parent = resolve_parent(body["parent"])
-
-          # Build state from auth context (if present)
-          state = build_state(env)
+          result = @service.validate(body)
+          return error_response(result[:status], result[:error]) if result[:error]
 
           # The protocol-rack adapter (used by Falcon and any server built on
           # protocol-http) buffers `each`-based bodies through an Enumerable path.
@@ -50,64 +27,7 @@ module Braintrust
           body_class = env.key?("protocol.http.request") ? SSEStreamBody : SSEBody
 
           sse_body = body_class.new do |sse|
-            # Only pass project/experiment params when state is available
-            run_opts = {
-              on_progress: ->(progress_data) {
-                # Build remote eval protocol events from generic progress data.
-                # Runner provides: id, data/error, scores (optional), origin (optional).
-                # Protocol requires: id, object_type, origin, name, format, output_type, event, data.
-                base = {
-                  "object_type" => "task",
-                  "name" => name,
-                  "format" => "code",
-                  "output_type" => "completion"
-                }
-                base["id"] = progress_data["id"] if progress_data["id"]
-                base["origin"] = progress_data["origin"] if progress_data["origin"]
-
-                if progress_data.key?("error")
-                  sse.event("progress", JSON.dump(base.merge("event" => "error", "data" => progress_data["error"])))
-                else
-                  sse.event("progress", JSON.dump(base.merge("event" => "json_delta", "data" => JSON.dump(progress_data["data"]))))
-                end
-
-                # Signal per-cell completion so the UI exits "Streaming..." state
-                # and updates the progress bar immediately.
-                sse.event("progress", JSON.dump(base.merge("event" => "done", "data" => "")))
-              },
-              quiet: true
-            }
-            run_opts[:parent] = parent if parent
-            run_opts[:scorers] = remote_scorer_ids if remote_scorer_ids
-            run_opts[:dataset] = dataset if dataset
-
-            if state
-              run_opts[:state] = state
-              run_opts[:experiment] = experiment_name if experiment_name
-              run_opts[:project_id] = body["project_id"] if body["project_id"]
-            end
-
-            result = evaluator.run(cases, **run_opts)
-
-            # Flush buffered OTLP spans before sending completion events.
-            # The BatchSpanProcessor exports every ~5s; fast evals can finish
-            # before a single export fires, causing the UI to see no results.
-            Braintrust::Trace.flush_spans
-
-            # Build summary from result scores
-            averaged_scores = {}
-            result.scorer_stats.each do |scorer_name, stats|
-              averaged_scores[scorer_name] = stats.score_mean
-            end
-
-            sse.event("summary", JSON.dump({
-              "scores" => averaged_scores,
-              "experiment_name" => experiment_name,
-              "experiment_id" => result.experiment_id,
-              "project_id" => result.project_id
-            }))
-
-            sse.event("done", "")
+            @service.stream(result, auth: env["braintrust.auth"], sse: sse)
           end
 
           [200, {"content-type" => "text/event-stream", "cache-control" => "no-cache", "connection" => "keep-alive"}, sse_body]
@@ -115,90 +35,6 @@ module Braintrust
 
         private
 
-        # Resolve data source from the data field.
-        # Returns [cases, dataset] where exactly one is non-nil.
-        def resolve_data_source(data)
-          if data.key?("data")
-            cases = data["data"].map do |d|
-              {input: d["input"], expected: d["expected"]}
-            end
-            [cases, nil]
-          elsif data.key?("dataset_id")
-            [nil, Braintrust::Dataset::ID.new(id: data["dataset_id"])]
-          elsif data.key?("dataset_name")
-            dataset_opts = {name: data["dataset_name"]}
-            dataset_opts[:project] = data["project_name"] if data["project_name"]
-            [nil, dataset_opts]
-          else
-            [nil, nil]
-          end
-        end
-
-        # Map request scores array to Scorer::ID structs.
-        # The UI sends function_id as a nested object: {"function_id": "uuid"}.
-        def resolve_remote_scorers(scores)
-          return nil if scores.nil? || scores.empty?
-          scores.map do |s|
-            func_id = s["function_id"]
-            func_id = func_id["function_id"] if func_id.is_a?(Hash)
-            Braintrust::Scorer::ID.new(
-              function_id: func_id,
-              version: s["version"]
-            )
-          end
-        end
-
-        # Map request parent to symbol-keyed Hash.
-        # Hardcode playground_id to match Java SDK behavior.
-        # Also extracts generation from propagated_event for span_attributes.
-        def resolve_parent(parent)
-          return nil unless parent.is_a?(Hash)
-          object_id = parent["object_id"]
-          return nil unless object_id
-
-          generation = parent.dig("propagated_event", "span_attributes", "generation")
-
-          result = {object_type: "playground_id", object_id: object_id}
-          result[:generation] = generation if generation
-          result
-        end
-
-        # Build State from auth context set by Auth middleware.
-        # Returns nil when no auth context is present (e.g. NoAuth strategy).
-        # Uses an LRU-style cache (max 64 entries) keyed by [api_key, app_url, org_name].
-        def build_state(env)
-          auth = env["braintrust.auth"]
-          return nil unless auth.is_a?(Hash)
-
-          cache_key = [auth["api_key"], auth["app_url"], auth["org_name"]]
-
-          @state_mutex ||= Mutex.new
-          @state_cache ||= {}
-
-          @state_mutex.synchronize do
-            cached = @state_cache[cache_key]
-            return cached if cached
-
-            state = Braintrust::State.new(
-              api_key: auth["api_key"],
-              org_id: auth["org_id"],
-              org_name: auth["org_name"],
-              app_url: auth["app_url"],
-              api_url: auth["api_url"],
-              enable_tracing: false
-            )
-
-            # Evict oldest entry if cache is full
-            if @state_cache.size >= 64
-              oldest_key = @state_cache.keys.first
-              @state_cache.delete(oldest_key)
-            end
-
-            @state_cache[cache_key] = state
-            state
-          end
-        end
-
         def parse_body(env)
           body = env["rack.input"]&.read
           return nil if body.nil? || body.empty?
@@ -211,6 +47,10 @@ module Braintrust
           [status, {"content-type" => "application/json"},
             [JSON.dump({"error" => message})]]
         end
+
+        def build_state(env)
+          @service.build_state(env["braintrust.auth"])
+        end
       end
     end
   end

data/lib/braintrust/server/handlers/list.rb

@@ -23,50 +23,12 @@ module Braintrust
       class List
         def initialize(evaluators)
           @evaluators = evaluators
+          @service = Services::List.new(evaluators)
         end
 
         def call(_env)
-          result = {}
-          @evaluators.each do |name, evaluator|
-            scores = (evaluator.scorers || []).each_with_index.map do |scorer, i|
-              scorer_name = scorer.respond_to?(:name) ? scorer.name : "score_#{i}"
-              {"name" => scorer_name}
-            end
-            entry = {"scores" => scores}
-            params = serialize_parameters(evaluator.parameters)
-            entry["parameters"] = params if params
-            result[name] = entry
-          end
-
-          [200, {"content-type" => "application/json"},
-            [JSON.dump(result)]]
-        end
-
-        private
-
-        # Convert user-defined parameters to the dev server protocol format.
-        # Wraps in a staticParameters container with "data" typed entries.
-        def serialize_parameters(parameters)
-          return nil unless parameters && !parameters.empty?
-
-          schema = {}
-          parameters.each do |name, spec|
-            spec = spec.transform_keys(&:to_s) if spec.is_a?(Hash)
-            if spec.is_a?(Hash)
-              schema[name.to_s] = {
-                "type" => "data",
-                "schema" => {"type" => spec["type"] || "string"},
-                "default" => spec["default"],
-                "description" => spec["description"]
-              }
-            end
-          end
-
-          {
-            "type" => "braintrust.staticParameters",
-            "schema" => schema,
-            "source" => nil
-          }
+          result = @service.call
+          [200, {"content-type" => "application/json"}, [JSON.dump(result)]]
         end
       end
     end

data/lib/braintrust/server/rack.rb

@@ -15,6 +15,8 @@ require_relative "auth/no_auth"
 require_relative "auth/clerk_token"
 require_relative "middleware/cors"
 require_relative "middleware/auth"
+require_relative "services/list_service"
+require_relative "services/eval_service"
 require_relative "handlers/health"
 require_relative "handlers/list"
 require_relative "handlers/eval"

data/lib/braintrust/server/services/eval_service.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Braintrust
+  module Server
+    module Services
+      # Framework-agnostic service for running evaluations and streaming SSE results.
+      # Must be long-lived (not per-request) to preserve the @state_cache across requests.
+      class Eval
+        def initialize(evaluators)
+          @evaluators = evaluators
+          @state_mutex = Mutex.new
+          @state_cache = {}
+        end
+
+        # Validates request body. Returns:
+        #   {error: String, status: Integer} on failure
+        #   {evaluator:, name:, cases:, dataset:, ...} on success
+        def validate(body)
+          name = body["name"]
+          return {error: "Missing required field: name", status: 400} unless name
+
+          evaluator = current_evaluators[name]
+          return {error: "Evaluator '#{name}' not found", status: 404} unless evaluator
+
+          data = body["data"]
+          return {error: "Missing required field: data", status: 400} unless data
+
+          data_sources = ["data", "dataset_name", "dataset_id"].count { |k| data.key?(k) }
+          return {error: "Exactly one data source required", status: 400} if data_sources != 1
+
+          cases, dataset = resolve_data_source(data)
+
+          {
+            evaluator: evaluator,
+            name: name,
+            cases: cases,
+            dataset: dataset,
+            experiment_name: body["experiment_name"],
+            remote_scorer_ids: resolve_remote_scorers(body["scores"]),
+            parent: resolve_parent(body["parent"]),
+            project_id: body["project_id"],
+            parameters: resolve_parameters(body["parameters"], evaluator)
+          }
+        end
+
+        # Runs the validated eval and streams SSE events via the sse writer.
+        # +validated+ is the hash returned by #validate.
+        # +auth+ is the auth context hash (or nil/true for no-auth).
+        # +sse+ is an SSEWriter instance.
+        def stream(validated, auth:, sse:)
+          name = validated[:name]
+          evaluator = validated[:evaluator]
+          cases = validated[:cases]
+          dataset = validated[:dataset]
+          experiment_name = validated[:experiment_name]
+          remote_scorer_ids = validated[:remote_scorer_ids]
+          parent = validated[:parent]
+          project_id = validated[:project_id]
+          parameters = validated[:parameters]
+
+          state = build_state(auth)
+
+          # Only pass project/experiment params when state is available
+          run_opts = {
+            on_progress: ->(progress_data) {
+              # Build remote eval protocol events from generic progress data.
+              # Runner provides: id, data/error, scores (optional), origin (optional).
+              # Protocol requires: id, object_type, origin, name, format, output_type, event, data.
+              base = {
+                "object_type" => "task",
+                "name" => name,
+                "format" => "code",
+                "output_type" => "completion"
+              }
+              base["id"] = progress_data["id"] if progress_data["id"]
+              base["origin"] = progress_data["origin"] if progress_data["origin"]
+
+              if progress_data.key?("error")
+                sse.event("progress", JSON.dump(base.merge("event" => "error", "data" => progress_data["error"])))
+              else
+                sse.event("progress", JSON.dump(base.merge("event" => "json_delta", "data" => JSON.dump(progress_data["data"]))))
+              end
+
+              # Signal per-cell completion so the UI exits "Streaming..." state
+              # and updates the progress bar immediately.
+              sse.event("progress", JSON.dump(base.merge("event" => "done", "data" => "")))
+            },
+            quiet: true
+          }
+          run_opts[:parent] = parent if parent
+          run_opts[:scorers] = remote_scorer_ids if remote_scorer_ids
+          run_opts[:parameters] = parameters if parameters && !parameters.empty?
+          run_opts[:dataset] = dataset if dataset
+
+          if state
+            run_opts[:state] = state
+            run_opts[:experiment] = experiment_name if experiment_name
+            run_opts[:project_id] = project_id if project_id
+          end
+
+          result = evaluator.run(cases, **run_opts)
+
+          # Flush buffered OTLP spans before sending completion events.
+          # The BatchSpanProcessor exports every ~5s; fast evals can finish
+          # before a single export fires, causing the UI to see no results.
+          Braintrust::Trace.flush_spans
+
+          # Build summary from result scores
+          averaged_scores = {}
+          result.scorer_stats.each do |scorer_name, stats|
+            averaged_scores[scorer_name] = stats.score_mean
+          end
+
+          sse.event("summary", JSON.dump({
+            "scores" => averaged_scores,
+            "experiment_name" => experiment_name,
+            "experiment_id" => result.experiment_id,
+            "project_id" => result.project_id
+          }))
+
+          sse.event("done", "")
+        end
+
+        # Build State from auth context hash.
+        # Returns nil when auth is not a Hash (e.g. NoAuth returns true).
+        # Uses an LRU-style cache (max 64 entries) keyed by [api_key, app_url, org_name].
+        def build_state(auth)
+          return nil unless auth.is_a?(Hash)
+
+          cache_key = [auth["api_key"], auth["app_url"], auth["org_name"]]
+
+          @state_mutex ||= Mutex.new
+          @state_cache ||= {}
+
+          @state_mutex.synchronize do
+            cached = @state_cache[cache_key]
+            return cached if cached
+
+            state = Braintrust::State.new(
+              api_key: auth["api_key"],
+              org_id: auth["org_id"],
+              org_name: auth["org_name"],
+              app_url: auth["app_url"],
+              api_url: auth["api_url"],
+              enable_tracing: false
+            )
+
+            if @state_cache.size >= 64
+              oldest_key = @state_cache.keys.first
+              @state_cache.delete(oldest_key)
+            end
+
+            @state_cache[cache_key] = state
+            state
+          end
+        end
+
+        private
+
+        def current_evaluators
+          return @evaluators.call if @evaluators.respond_to?(:call)
+          @evaluators
+        end
+
+        # Merge request parameters with evaluator's parameter defaults.
+        # Request values override defaults. Returns a string-keyed Hash.
+        def resolve_parameters(raw_params, evaluator)
+          defaults = (evaluator.parameters || {}).to_h { |name, spec|
+            [name.to_s, spec.is_a?(Hash) ? (spec[:default] || spec["default"]) : nil]
+          }.compact
+          defaults.merge(raw_params || {})
+        end
+
+        # Resolve data source from the data field.
+        # Returns [cases, dataset] where exactly one is non-nil.
+        def resolve_data_source(data)
+          if data.key?("data")
+            cases = data["data"].map do |d|
+              {input: d["input"], expected: d["expected"]}
+            end
+            [cases, nil]
+          elsif data.key?("dataset_id")
+            [nil, Braintrust::Dataset::ID.new(id: data["dataset_id"])]
+          elsif data.key?("dataset_name")
+            dataset_opts = {name: data["dataset_name"]}
+            dataset_opts[:project] = data["project_name"] if data["project_name"]
+            [nil, dataset_opts]
+          else
+            [nil, nil]
+          end
+        end
+
+        # Map request scores array to Scorer::ID structs.
+        # The UI sends function_id as a nested object: {"function_id": "uuid"}.
+        def resolve_remote_scorers(scores)
+          return nil if scores.nil? || scores.empty?
+          scores.map do |s|
+            func_id = s["function_id"]
+            func_id = func_id["function_id"] if func_id.is_a?(Hash)
+            Braintrust::Scorer::ID.new(
+              function_id: func_id,
+              version: s["version"]
+            )
+          end
+        end
+
+        # Map request parent to symbol-keyed Hash.
+        # Hardcode playground_id to match Java SDK behavior.
+        # Also extracts generation from propagated_event for span_attributes.
+        def resolve_parent(parent)
+          return nil unless parent.is_a?(Hash)
+          object_id = parent["object_id"]
+          return nil unless object_id
+
+          generation = parent.dig("propagated_event", "span_attributes", "generation")
+
+          result = {object_type: "playground_id", object_id: object_id}
+          result[:generation] = generation if generation
+          result
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/server/services/list_service.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Braintrust
+  module Server
+    module Services
+      # Framework-agnostic service for listing evaluators.
+      # Returns a plain Hash (not a Rack triplet) suitable for JSON.dump.
+      class List
+        def initialize(evaluators)
+          @evaluators = evaluators
+        end
+
+        def call
+          result = {}
+          current_evaluators.each do |name, evaluator|
+            scores = (evaluator.scorers || []).each_with_index.map do |scorer, i|
+              scorer_name = scorer.respond_to?(:name) ? scorer.name : "score_#{i}"
+              {"name" => scorer_name}
+            end
+            entry = {"scores" => scores}
+            params = serialize_parameters(evaluator.parameters)
+            entry["parameters"] = params if params
+            result[name] = entry
+          end
+          result
+        end
+
+        private
+
+        def current_evaluators
+          return @evaluators.call if @evaluators.respond_to?(:call)
+          @evaluators
+        end
+
+        # Convert user-defined parameters to the dev server protocol format.
+        # Wraps in a staticParameters container with "data" typed entries.
+        def serialize_parameters(parameters)
+          return nil unless parameters && !parameters.empty?
+
+          schema = {}
+          parameters.each do |name, spec|
+            spec = spec.transform_keys(&:to_s) if spec.is_a?(Hash)
+            if spec.is_a?(Hash)
+              schema[name.to_s] = {
+                "type" => "data",
+                "schema" => {"type" => spec["type"] || "string"},
+                "default" => spec["default"],
+                "description" => spec["description"]
+              }
+            end
+          end
+
+          {
+            "type" => "braintrust.staticParameters",
+            "schema" => schema,
+            "source" => nil
+          }
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/trace/span_processor.rb

@@ -80,11 +80,6 @@ module Braintrust
       # Determine if a span should be forwarded to the wrapped processor
       # based on configured filters
       def should_forward_span?(span)
-        # Always keep root spans (spans with no parent)
-        # Check if parent_span_id is the invalid/zero span ID
-        is_root = span.parent_span_id == OpenTelemetry::Trace::INVALID_SPAN_ID
-        return true if is_root
-
         # If no filters, keep everything
         return true if @filters.empty?
 

data/lib/braintrust/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Braintrust
-  VERSION = "0.2.1"
+  VERSION = "0.3.1"
 end