braintrust 0.1.4 → 0.2.0

data/lib/braintrust/eval.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 require_relative "eval/scorer"
+require_relative "eval/evaluator"
 require_relative "eval/runner"
+require_relative "eval/functions"
 require_relative "api/internal/projects"
 require_relative "api/internal/experiments"
 require_relative "dataset"
@@ -186,14 +188,17 @@ module Braintrust
       end
 
       # Run an evaluation
-      # @param project [String] The project name
-      # @param experiment [String] The experiment name
+      # @param project [String, nil] The project name (triggers full API mode: creates project + experiment)
+      # @param experiment [String, nil] The experiment name
       # @param cases [Array, Enumerable, nil] The test cases (mutually exclusive with dataset)
       # @param dataset [String, Hash, nil] Dataset to fetch (mutually exclusive with cases)
       #   - String: dataset name (fetches from same project)
       #   - Hash: {name:, id:, project:, version:, limit:}
       # @param task [#call] The task to evaluate (must be callable)
       # @param scorers [Array<Scorer, #call>] The scorers to use (Scorer objects or callables)
+      # @param on_progress [#call, nil] Optional callback fired after each test case.
+      #   Receives a Hash: {"data" => output, "scores" => {name => value}} on success,
+      #   or {"error" => message} on failure.
       # @param parallelism [Integer] Number of parallel workers (default: 1).
       #   When parallelism > 1, test cases are executed concurrently using a thread pool.
       #   The task and scorers MUST be thread-safe when using parallelism > 1.
@@ -201,53 +206,45 @@ module Braintrust
       # @param metadata [Hash] Optional experiment metadata
       # @param update [Boolean] If true, allow reusing existing experiment (default: false)
       # @param quiet [Boolean] If true, suppress result output (default: false)
-      # @param api [API, nil] Braintrust API client (defaults to API.new using global state)
+      # @param state [State, nil] Braintrust state (defaults to global state)
       # @param tracer_provider [TracerProvider, nil] OpenTelemetry tracer provider (defaults to global)
       # @return [Result]
-      def run(project:, experiment:, task:, scorers:,
-        cases: nil, dataset: nil,
+      def run(task:, scorers:, project: nil, experiment: nil,
+        cases: nil, dataset: nil, on_progress: nil,
         parallelism: 1, tags: nil, metadata: nil, update: false, quiet: false,
-        api: nil, tracer_provider: nil)
+        state: nil, tracer_provider: nil, project_id: nil, parent: nil)
         # Validate required parameters
-        validate_params!(project: project, experiment: experiment,
-          cases: cases, dataset: dataset, task: task, scorers: scorers)
+        validate_params!(task: task, scorers: scorers, cases: cases, dataset: dataset)
 
-        # Get API from parameter or create from global state
-        api ||= API.new
+        # Resolve any ScorerId entries to real Scorer objects
+        scorers = resolve_scorers(scorers, state: state, tracer_provider: tracer_provider)
 
-        # Ensure logged in (to populate org_name, etc.)
-        # login is idempotent and returns early if already logged in
-        api.login
+        experiment_id = nil
+        project_name = project
 
-        # Resolve dataset to cases if dataset parameter provided
-        dataset_id = nil
-        dataset_version = nil
+        # Full API mode: project name or project_id provided, resolve via API
+        if project || project_id
+          state ||= Braintrust.current_state
+          state.login
 
-        if dataset
-          resolved = resolve_dataset(dataset, project, api)
-          cases = resolved[:cases]
-          dataset_id = resolved[:dataset_id]
-          dataset_version = resolved[:dataset_version]
-        end
-
-        # Register project and experiment via internal API
-        projects_api = API::Internal::Projects.new(api.state)
-        experiments_api = API::Internal::Experiments.new(api.state)
+          if dataset
+            resolved = resolve_dataset(dataset, project, state)
+            cases = resolved[:cases]
+          end
 
-        project_result = projects_api.create(name: project)
-        experiment_result = experiments_api.create(
-          name: experiment,
-          project_id: project_result["id"],
-          ensure_new: !update,
-          tags: tags,
-          metadata: metadata,
-          dataset_id: dataset_id,
-          dataset_version: dataset_version
-        )
-
-        experiment_id = experiment_result["id"]
-        project_id = project_result["id"]
-        project_name = project_result["name"]
+          # Skip experiment creation for remote evals (parent present).
+          # The OTLP backend creates experiments from ingested spans.
+          unless parent
+            project_id, project_name = resolve_project(state, project, project_id)
+            experiment_id = create_experiment(
+              state, experiment, project_id,
+              update: update, tags: tags, metadata: metadata,
+              dataset_id: resolved&.dig(:dataset_id),
+              dataset_version: resolved&.dig(:dataset_version)
+            )
+            parent = {object_type: "experiment_id", object_id: experiment_id}
+          end
+        end
 
         # Instantiate Runner and run evaluation
         runner = Runner.new(
@@ -257,8 +254,10 @@ module Braintrust
           project_name: project_name,
           task: task,
           scorers: scorers,
-          api: api,
-          tracer_provider: tracer_provider
+          state: state,
+          tracer_provider: tracer_provider,
+          on_progress: on_progress,
+          parent: parent
         )
         result = runner.run(cases, parallelism: parallelism)
 
@@ -276,11 +275,29 @@ module Braintrust
         puts result.to_pretty
       end
 
+      # Resolve scorers array: ScorerId entries become real Scorer objects, others pass through
+      # @param scorers [Array] Scorers (Scorer, callable, or ScorerId)
+      # @param state [State, nil] Braintrust state (required for ScorerId resolution)
+      # @param tracer_provider [TracerProvider, nil] OpenTelemetry tracer provider
+      # @return [Array<Scorer, #call>] Resolved scorers
+      def resolve_scorers(scorers, state: nil, tracer_provider: nil)
+        scorers.map do |scorer|
+          if scorer.is_a?(ScorerId)
+            Functions.scorer_by_id(
+              id: scorer.function_id,
+              version: scorer.version,
+              state: state,
+              tracer_provider: tracer_provider
+            )
+          else
+            scorer
+          end
+        end
+      end
+
       # Validate required parameters
       # @raise [ArgumentError] if validation fails
-      def validate_params!(project:, experiment:, cases:, dataset:, task:, scorers:)
-        raise ArgumentError, "project is required" unless project
-        raise ArgumentError, "experiment is required" unless experiment
+      def validate_params!(task:, scorers:, cases:, dataset:)
         raise ArgumentError, "task is required" unless task
         raise ArgumentError, "scorers is required" unless scorers
 
@@ -300,27 +317,57 @@ module Braintrust
         end
       end
 
+      # Resolve project by name or ID. Creates if needed.
+      # @return [Array(String, String)] [project_id, project_name]
+      def resolve_project(state, project, project_id)
+        if project_id
+          [project_id, project]
+        else
+          result = API::Internal::Projects.new(state).create(name: project)
+          [result["id"], result["name"]]
+        end
+      end
+
+      # Create an experiment in the given project.
+      # @return [String] experiment_id
+      def create_experiment(state, name, project_id,
+        update: false, tags: nil, metadata: nil,
+        dataset_id: nil, dataset_version: nil)
+        result = API::Internal::Experiments.new(state).create(
+          name: name,
+          project_id: project_id,
+          ensure_new: !update,
+          tags: tags,
+          metadata: metadata,
+          dataset_id: dataset_id,
+          dataset_version: dataset_version
+        )
+        result["id"]
+      end
+
       # Resolve dataset parameter to cases with metadata for experiment linking
       # @param dataset [String, Hash, Dataset] Dataset specifier or instance
       # @param project [String] Project name (used as default if not specified)
-      # @param api [API] Braintrust API client
+      # @param state [State] Braintrust state
       # @return [Hash] Hash with :cases, :dataset_id, and :dataset_version
-      def resolve_dataset(dataset, project, api)
+      def resolve_dataset(dataset, project, state)
         limit = nil
 
         dataset_obj = case dataset
         when Dataset
           dataset
+        when DatasetId
+          Dataset.new(id: dataset.id, state: state)
         when String
-          Dataset.new(name: dataset, project: project, api: api)
+          Dataset.new(name: dataset, project: project, state: state)
         when Hash
           opts = dataset.dup
           limit = opts.delete(:limit)
           opts[:project] ||= project
-          opts[:api] = api
+          opts[:state] = state
           Dataset.new(**opts)
         else
-          raise ArgumentError, "dataset must be String, Hash, or Dataset, got #{dataset.class}"
+          raise ArgumentError, "dataset must be String, Hash, Dataset, or DatasetId, got #{dataset.class}"
         end
 
         cases = dataset_obj.fetch_all(limit: limit)

data/lib/braintrust/server/auth/clerk_token.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+
+module Braintrust
+  module Server
+    module Auth
+      # Validates Clerk JWT session tokens via the Braintrust app endpoint.
+      # The browser forwards the Clerk session token which is validated by
+      # POST /api/apikey/login on the app server.
+      class ClerkToken
+        DEFAULT_APP_URL = "https://www.braintrust.dev"
+        RACK_AUTH_HEADER = "HTTP_AUTHORIZATION"
+        RACK_ORG_NAME_HEADER = "HTTP_X_BT_ORG_NAME"
+        BEARER_PATTERN = /\ABearer (.+)\z/
+        LOGIN_PATH = "/api/apikey/login"
+
+        def initialize(app_url: nil)
+          @app_url = app_url || DEFAULT_APP_URL
+        end
+
+        def authenticate(env)
+          token = extract_bearer_token(env)
+          return nil unless token
+
+          login_response = validate_token(token)
+          return nil unless login_response
+
+          org_name = env[RACK_ORG_NAME_HEADER]
+
+          {
+            "api_key" => token,
+            "org_id" => login_response["org_id"],
+            "org_name" => org_name || login_response["org_name"],
+            "app_url" => @app_url,
+            "api_url" => login_response["api_url"] || @app_url
+          }
+        end
+
+        private
+
+        def extract_bearer_token(env)
+          header = env[RACK_AUTH_HEADER]
+          return nil unless header
+          header[BEARER_PATTERN, 1]
+        end
+
+        def validate_token(token)
+          uri = URI("#{@app_url}#{LOGIN_PATH}")
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = uri.scheme == "https"
+
+          request = Net::HTTP::Post.new(uri)
+          request["Content-Type"] = "application/json"
+          request.body = JSON.dump({token: token})
+
+          response = http.request(request)
+          return nil unless response.code == "200"
+
+          JSON.parse(response.body)
+        rescue
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/server/auth/no_auth.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Server
+    module Auth
+      # No-op auth strategy for testing and local development.
+      class NoAuth
+        def authenticate(_env)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/server/handlers/eval.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Braintrust
+  module Server
+    module Handlers
+      # POST /eval — adapter that maps HTTP request to Evaluator#run and streams SSE results.
+      # Handles auth passthrough, datasets, remote scorers, project_id, and parent.
+      class Eval
+        def initialize(evaluators)
+          @evaluators = evaluators
+        end
+
+        def call(env)
+          body = parse_body(env)
+          return error_response(400, "Invalid JSON body") unless body
+
+          name = body["name"]
+          return error_response(400, "Missing required field: name") unless name
+
+          evaluator = @evaluators[name]
+          return error_response(404, "Evaluator '#{name}' not found") unless evaluator
+
+          data = body["data"]
+          return error_response(400, "Missing required field: data") unless data
+
+          # Validate exactly one data source
+          data_sources = ["data", "dataset_name", "dataset_id"].count { |k| data.key?(k) }
+          return error_response(400, "Exactly one data source required") if data_sources != 1
+
+          experiment_name = body["experiment_name"]
+
+          # Resolve data source
+          cases, dataset = resolve_data_source(data)
+
+          # Resolve remote scorers from request
+          remote_scorer_ids = resolve_remote_scorers(body["scores"])
+
+          # Resolve parent span context
+          parent = resolve_parent(body["parent"])
+
+          # Build state from auth context (if present)
+          state = build_state(env)
+
+          # The protocol-rack adapter (used by Falcon and any server built on
+          # protocol-http) buffers `each`-based bodies through an Enumerable path.
+          # Detect it via the "protocol.http.request" env key it injects, and use
+          # SSEStreamBody (call-only) so it dispatches through the Streaming path.
+          body_class = env.key?("protocol.http.request") ? SSEStreamBody : SSEBody
+
+          sse_body = body_class.new do |sse|
+            # Only pass project/experiment params when state is available
+            run_opts = {
+              on_progress: ->(progress_data) {
+                # Build remote eval protocol events from generic progress data.
+                # Runner provides: id, data/error, scores (optional), origin (optional).
+                # Protocol requires: id, object_type, origin, name, format, output_type, event, data.
+                base = {
+                  "object_type" => "task",
+                  "name" => name,
+                  "format" => "code",
+                  "output_type" => "completion"
+                }
+                base["id"] = progress_data["id"] if progress_data["id"]
+                base["origin"] = progress_data["origin"] if progress_data["origin"]
+
+                if progress_data.key?("error")
+                  sse.event("progress", JSON.dump(base.merge("event" => "error", "data" => progress_data["error"])))
+                else
+                  sse.event("progress", JSON.dump(base.merge("event" => "json_delta", "data" => JSON.dump(progress_data["data"]))))
+                end
+
+                # Signal per-cell completion so the UI exits "Streaming..." state
+                # and updates the progress bar immediately.
+                sse.event("progress", JSON.dump(base.merge("event" => "done", "data" => "")))
+              },
+              quiet: true
+            }
+            run_opts[:parent] = parent if parent
+            run_opts[:scorers] = remote_scorer_ids if remote_scorer_ids
+            run_opts[:dataset] = dataset if dataset
+
+            if state
+              run_opts[:state] = state
+              run_opts[:experiment] = experiment_name if experiment_name
+              run_opts[:project_id] = body["project_id"] if body["project_id"]
+            end
+
+            result = evaluator.run(cases, **run_opts)
+
+            # Flush buffered OTLP spans before sending completion events.
+            # The BatchSpanProcessor exports every ~5s; fast evals can finish
+            # before a single export fires, causing the UI to see no results.
+            Braintrust::Trace.flush_spans
+
+            # Build summary from result scores
+            averaged_scores = {}
+            result.scorer_stats.each do |scorer_name, stats|
+              averaged_scores[scorer_name] = stats.score_mean
+            end
+
+            sse.event("summary", JSON.dump({
+              "scores" => averaged_scores,
+              "experiment_name" => experiment_name,
+              "experiment_id" => result.experiment_id,
+              "project_id" => result.project_id
+            }))
+
+            sse.event("done", "")
+          end
+
+          [200, {"content-type" => "text/event-stream", "cache-control" => "no-cache", "connection" => "keep-alive"}, sse_body]
+        end
+
+        private
+
+        # Resolve data source from the data field.
+        # Returns [cases, dataset] where exactly one is non-nil.
+        def resolve_data_source(data)
+          if data.key?("data")
+            cases = data["data"].map do |d|
+              {input: d["input"], expected: d["expected"]}
+            end
+            [cases, nil]
+          elsif data.key?("dataset_id")
+            [nil, Braintrust::DatasetId.new(id: data["dataset_id"])]
+          elsif data.key?("dataset_name")
+            dataset_opts = {name: data["dataset_name"]}
+            dataset_opts[:project] = data["project_name"] if data["project_name"]
+            [nil, dataset_opts]
+          else
+            [nil, nil]
+          end
+        end
+
+        # Map request scores array to ScorerId structs.
+        # The UI sends function_id as a nested object: {"function_id": "uuid"}.
+        def resolve_remote_scorers(scores)
+          return nil if scores.nil? || scores.empty?
+          scores.map do |s|
+            func_id = s["function_id"]
+            func_id = func_id["function_id"] if func_id.is_a?(Hash)
+            Braintrust::ScorerId.new(
+              function_id: func_id,
+              version: s["version"]
+            )
+          end
+        end
+
+        # Map request parent to symbol-keyed Hash.
+        # Hardcode playground_id to match Java SDK behavior.
+        # Also extracts generation from propagated_event for span_attributes.
+        def resolve_parent(parent)
+          return nil unless parent.is_a?(Hash)
+          object_id = parent["object_id"]
+          return nil unless object_id
+
+          generation = parent.dig("propagated_event", "span_attributes", "generation")
+
+          result = {object_type: "playground_id", object_id: object_id}
+          result[:generation] = generation if generation
+          result
+        end
+
+        # Build State from auth context set by Auth middleware.
+        # Returns nil when no auth context is present (e.g. NoAuth strategy).
+        # Uses an LRU-style cache (max 64 entries) keyed by [api_key, app_url, org_name].
+        def build_state(env)
+          auth = env["braintrust.auth"]
+          return nil unless auth.is_a?(Hash)
+
+          cache_key = [auth["api_key"], auth["app_url"], auth["org_name"]]
+
+          @state_mutex ||= Mutex.new
+          @state_cache ||= {}
+
+          @state_mutex.synchronize do
+            cached = @state_cache[cache_key]
+            return cached if cached
+
+            state = Braintrust::State.new(
+              api_key: auth["api_key"],
+              org_id: auth["org_id"],
+              org_name: auth["org_name"],
+              app_url: auth["app_url"],
+              api_url: auth["api_url"],
+              enable_tracing: false
+            )
+
+            # Evict oldest entry if cache is full
+            if @state_cache.size >= 64
+              oldest_key = @state_cache.keys.first
+              @state_cache.delete(oldest_key)
+            end
+
+            @state_cache[cache_key] = state
+            state
+          end
+        end
+
+        def parse_body(env)
+          body = env["rack.input"]&.read
+          return nil if body.nil? || body.empty?
+          JSON.parse(body)
+        rescue JSON::ParserError
+          nil
+        end
+
+        def error_response(status, message)
+          [status, {"content-type" => "application/json"},
+            [JSON.dump({"error" => message})]]
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/server/handlers/health.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Braintrust
+  module Server
+    module Handlers
+      # GET / — simple health check endpoint.
+      class Health
+        def call(_env)
+          [200, {"content-type" => "application/json"}, [JSON.dump({"status" => "ok"})]]
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/server/handlers/list.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Braintrust
+  module Server
+    module Handlers
+      # GET/POST /list — returns all evaluators keyed by name.
+      #
+      # Response format (Braintrust dev server protocol):
+      #   {
+      #     "evaluator-name": {
+      #       "parameters": {                          # optional
+      #         "type": "braintrust.staticParameters",
+      #         "schema": {
+      #           "param_name": { "type": "data", "schema": {...}, "default": ..., "description": ... }
+      #         },
+      #         "source": null
+      #       },
+      #       "scores": [{ "name": "scorer_name" }, ...]
+      #     }
+      #   }
+      class List
+        def initialize(evaluators)
+          @evaluators = evaluators
+        end
+
+        def call(_env)
+          result = {}
+          @evaluators.each do |name, evaluator|
+            scores = (evaluator.scorers || []).each_with_index.map do |scorer, i|
+              scorer_name = scorer.respond_to?(:name) ? scorer.name : "score_#{i}"
+              {"name" => scorer_name}
+            end
+            entry = {"scores" => scores}
+            params = serialize_parameters(evaluator.parameters)
+            entry["parameters"] = params if params
+            result[name] = entry
+          end
+
+          [200, {"content-type" => "application/json"},
+            [JSON.dump(result)]]
+        end
+
+        private
+
+        # Convert user-defined parameters to the dev server protocol format.
+        # Wraps in a staticParameters container with "data" typed entries.
+        def serialize_parameters(parameters)
+          return nil unless parameters && !parameters.empty?
+
+          schema = {}
+          parameters.each do |name, spec|
+            spec = spec.transform_keys(&:to_s) if spec.is_a?(Hash)
+            if spec.is_a?(Hash)
+              schema[name.to_s] = {
+                "type" => "data",
+                "schema" => {"type" => spec["type"] || "string"},
+                "default" => spec["default"],
+                "description" => spec["description"]
+              }
+            end
+          end
+
+          {
+            "type" => "braintrust.staticParameters",
+            "schema" => schema,
+            "source" => nil
+          }
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/server/middleware/auth.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Braintrust
+  module Server
+    module Middleware
+      # Auth middleware that validates requests using a pluggable strategy.
+      # Sets env["braintrust.auth"] with the authentication result on success.
+      class Auth
+        def initialize(app, strategy:)
+          @app = app
+          @strategy = strategy
+        end
+
+        def call(env)
+          auth_result = @strategy.authenticate(env)
+          unless auth_result
+            return [401, {"content-type" => "application/json"},
+              [JSON.dump({"error" => "Unauthorized"})]]
+          end
+
+          env["braintrust.auth"] = auth_result
+          @app.call(env)
+        end
+      end
+    end
+  end
+end