braintrust 0.1.3 → 0.2.0

data/lib/braintrust/eval.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 require_relative "eval/scorer"
+require_relative "eval/evaluator"
 require_relative "eval/runner"
+require_relative "eval/functions"
 require_relative "api/internal/projects"
 require_relative "api/internal/experiments"
 require_relative "dataset"
@@ -186,14 +188,17 @@ module Braintrust
       end
 
       # Run an evaluation
-      # @param project [String] The project name
-      # @param experiment [String] The experiment name
+      # @param project [String, nil] The project name (triggers full API mode: creates project + experiment)
+      # @param experiment [String, nil] The experiment name
       # @param cases [Array, Enumerable, nil] The test cases (mutually exclusive with dataset)
       # @param dataset [String, Hash, nil] Dataset to fetch (mutually exclusive with cases)
       #   - String: dataset name (fetches from same project)
       #   - Hash: {name:, id:, project:, version:, limit:}
       # @param task [#call] The task to evaluate (must be callable)
       # @param scorers [Array<Scorer, #call>] The scorers to use (Scorer objects or callables)
+      # @param on_progress [#call, nil] Optional callback fired after each test case.
+      #   Receives a Hash: {"data" => output, "scores" => {name => value}} on success,
+      #   or {"error" => message} on failure.
       # @param parallelism [Integer] Number of parallel workers (default: 1).
       #   When parallelism > 1, test cases are executed concurrently using a thread pool.
       #   The task and scorers MUST be thread-safe when using parallelism > 1.
@@ -201,45 +206,45 @@ module Braintrust
       # @param metadata [Hash] Optional experiment metadata
       # @param update [Boolean] If true, allow reusing existing experiment (default: false)
       # @param quiet [Boolean] If true, suppress result output (default: false)
-      # @param api [API, nil] Braintrust API client (defaults to API.new using global state)
+      # @param state [State, nil] Braintrust state (defaults to global state)
       # @param tracer_provider [TracerProvider, nil] OpenTelemetry tracer provider (defaults to global)
       # @return [Result]
-      def run(project:, experiment:, task:, scorers:,
-        cases: nil, dataset: nil,
+      def run(task:, scorers:, project: nil, experiment: nil,
+        cases: nil, dataset: nil, on_progress: nil,
         parallelism: 1, tags: nil, metadata: nil, update: false, quiet: false,
-        api: nil, tracer_provider: nil)
+        state: nil, tracer_provider: nil, project_id: nil, parent: nil)
         # Validate required parameters
-        validate_params!(project: project, experiment: experiment,
-          cases: cases, dataset: dataset, task: task, scorers: scorers)
+        validate_params!(task: task, scorers: scorers, cases: cases, dataset: dataset)
 
-        # Get API from parameter or create from global state
-        api ||= API.new
+        # Resolve any ScorerId entries to real Scorer objects
+        scorers = resolve_scorers(scorers, state: state, tracer_provider: tracer_provider)
 
-        # Ensure logged in (to populate org_name, etc.)
-        # login is idempotent and returns early if already logged in
-        api.login
+        experiment_id = nil
+        project_name = project
 
-        # Resolve dataset to cases if dataset parameter provided
-        if dataset
-          cases = resolve_dataset(dataset, project, api)
-        end
-
-        # Register project and experiment via internal API
-        projects_api = API::Internal::Projects.new(api.state)
-        experiments_api = API::Internal::Experiments.new(api.state)
+        # Full API mode: project name or project_id provided, resolve via API
+        if project || project_id
+          state ||= Braintrust.current_state
+          state.login
 
-        project_result = projects_api.create(name: project)
-        experiment_result = experiments_api.create(
-          name: experiment,
-          project_id: project_result["id"],
-          ensure_new: !update,
-          tags: tags,
-          metadata: metadata
-        )
+          if dataset
+            resolved = resolve_dataset(dataset, project, state)
+            cases = resolved[:cases]
+          end
 
-        experiment_id = experiment_result["id"]
-        project_id = project_result["id"]
-        project_name = project_result["name"]
+          # Skip experiment creation for remote evals (parent present).
+          # The OTLP backend creates experiments from ingested spans.
+          unless parent
+            project_id, project_name = resolve_project(state, project, project_id)
+            experiment_id = create_experiment(
+              state, experiment, project_id,
+              update: update, tags: tags, metadata: metadata,
+              dataset_id: resolved&.dig(:dataset_id),
+              dataset_version: resolved&.dig(:dataset_version)
+            )
+            parent = {object_type: "experiment_id", object_id: experiment_id}
+          end
+        end
 
         # Instantiate Runner and run evaluation
         runner = Runner.new(
@@ -249,8 +254,10 @@ module Braintrust
           project_name: project_name,
           task: task,
           scorers: scorers,
-          api: api,
-          tracer_provider: tracer_provider
+          state: state,
+          tracer_provider: tracer_provider,
+          on_progress: on_progress,
+          parent: parent
         )
         result = runner.run(cases, parallelism: parallelism)
 
@@ -268,11 +275,29 @@ module Braintrust
         puts result.to_pretty
       end
 
+      # Resolve scorers array: ScorerId entries become real Scorer objects, others pass through
+      # @param scorers [Array] Scorers (Scorer, callable, or ScorerId)
+      # @param state [State, nil] Braintrust state (required for ScorerId resolution)
+      # @param tracer_provider [TracerProvider, nil] OpenTelemetry tracer provider
+      # @return [Array<Scorer, #call>] Resolved scorers
+      def resolve_scorers(scorers, state: nil, tracer_provider: nil)
+        scorers.map do |scorer|
+          if scorer.is_a?(ScorerId)
+            Functions.scorer_by_id(
+              id: scorer.function_id,
+              version: scorer.version,
+              state: state,
+              tracer_provider: tracer_provider
+            )
+          else
+            scorer
+          end
+        end
+      end
+
       # Validate required parameters
       # @raise [ArgumentError] if validation fails
-      def validate_params!(project:, experiment:, cases:, dataset:, task:, scorers:)
-        raise ArgumentError, "project is required" unless project
-        raise ArgumentError, "experiment is required" unless experiment
+      def validate_params!(task:, scorers:, cases:, dataset:)
         raise ArgumentError, "task is required" unless task
         raise ArgumentError, "scorers is required" unless scorers
 
@@ -292,30 +317,68 @@ module Braintrust
         end
       end
 
-      # Resolve dataset parameter to an array of case records
+      # Resolve project by name or ID. Creates if needed.
+      # @return [Array(String, String)] [project_id, project_name]
+      def resolve_project(state, project, project_id)
+        if project_id
+          [project_id, project]
+        else
+          result = API::Internal::Projects.new(state).create(name: project)
+          [result["id"], result["name"]]
+        end
+      end
+
+      # Create an experiment in the given project.
+      # @return [String] experiment_id
+      def create_experiment(state, name, project_id,
+        update: false, tags: nil, metadata: nil,
+        dataset_id: nil, dataset_version: nil)
+        result = API::Internal::Experiments.new(state).create(
+          name: name,
+          project_id: project_id,
+          ensure_new: !update,
+          tags: tags,
+          metadata: metadata,
+          dataset_id: dataset_id,
+          dataset_version: dataset_version
+        )
+        result["id"]
+      end
+
+      # Resolve dataset parameter to cases with metadata for experiment linking
       # @param dataset [String, Hash, Dataset] Dataset specifier or instance
       # @param project [String] Project name (used as default if not specified)
-      # @param api [API] Braintrust API client
-      # @return [Array<Hash>] Array of case records
-      def resolve_dataset(dataset, project, api)
+      # @param state [State] Braintrust state
+      # @return [Hash] Hash with :cases, :dataset_id, and :dataset_version
+      def resolve_dataset(dataset, project, state)
         limit = nil
 
         dataset_obj = case dataset
         when Dataset
           dataset
+        when DatasetId
+          Dataset.new(id: dataset.id, state: state)
         when String
-          Dataset.new(name: dataset, project: project, api: api)
+          Dataset.new(name: dataset, project: project, state: state)
         when Hash
           opts = dataset.dup
           limit = opts.delete(:limit)
           opts[:project] ||= project
-          opts[:api] = api
+          opts[:state] = state
           Dataset.new(**opts)
         else
-          raise ArgumentError, "dataset must be String, Hash, or Dataset, got #{dataset.class}"
+          raise ArgumentError, "dataset must be String, Hash, Dataset, or DatasetId, got #{dataset.class}"
         end
 
-        dataset_obj.fetch_all(limit: limit)
+        cases = dataset_obj.fetch_all(limit: limit)
+
+        # Use pinned version if available, otherwise compute from max(_xact_id)
+        version = dataset_obj.version
+        version ||= cases
+          .filter_map { |c| c[:origin] && JSON.parse(c[:origin])["_xact_id"] }
+          .max
+
+        {cases: cases, dataset_id: dataset_obj.id, dataset_version: version}
       end
     end
   end

data/lib/braintrust/internal/http.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "zlib"
+require "stringio"
+require_relative "../logger"
+
+module Braintrust
+  module Internal
+    # HTTP utilities for redirect following and response decompression.
+    # Drop-in enhancement for raw Net::HTTP request calls throughout the SDK.
+    module Http
+      DEFAULT_MAX_REDIRECTS = 5
+
+      # Execute an HTTP request, following redirects as needed.
+      #
+      # @param uri [URI] The request URI
+      # @param request [Net::HTTPRequest] The prepared request object
+      # @param max_redirects [Integer] Maximum number of redirects to follow
+      # @return [Net::HTTPResponse] The final response
+      # @raise [Braintrust::Error] On too many redirects or missing Location header
+      def self.with_redirects(uri, request, max_redirects: DEFAULT_MAX_REDIRECTS)
+        response = perform_request(uri, request)
+
+        redirects = 0
+        original_request = request
+
+        while response.is_a?(Net::HTTPRedirection)
+          redirects += 1
+          if redirects > max_redirects
+            raise Error, "Too many redirects (max #{max_redirects})"
+          end
+
+          location = response["location"]
+          unless location
+            raise Error, "Redirect response #{response.code} without Location header"
+          end
+
+          redirect_uri = URI(location)
+          redirect_uri = uri + redirect_uri unless redirect_uri.host
+
+          Log.debug("[HTTP] Following #{response.code} redirect to #{redirect_uri}")
+
+          request = build_redirect_request(response, redirect_uri, original_request, uri)
+          uri = redirect_uri
+          response = perform_request(uri, request)
+        end
+
+        response
+      end
+
+      # Decompress an HTTP response body in place based on Content-Encoding.
+      # No-op if the response has no recognized encoding.
+      #
+      # @param response [Net::HTTPResponse] The response to decompress
+      # @return [void]
+      def self.decompress_response!(response)
+        encoding = response["content-encoding"]&.downcase
+        case encoding
+        when "gzip", "x-gzip"
+          gz = Zlib::GzipReader.new(StringIO.new(response.body))
+          response.body.replace(gz.read)
+          gz.close
+          response.delete("content-encoding")
+        end
+      end
+
+      def self.perform_request(uri, request)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = (uri.scheme == "https")
+        http.request(request)
+      end
+      private_class_method :perform_request
+
+      def self.build_redirect_request(response, redirect_uri, original_request, original_uri)
+        if response.code == "307" || response.code == "308"
+          request = original_request.class.new(redirect_uri)
+          request.body = original_request.body
+          request["Content-Type"] = original_request["Content-Type"] if original_request["Content-Type"]
+        else
+          # 301, 302, 303: follow with GET, no body
+          request = Net::HTTP::Get.new(redirect_uri)
+        end
+
+        # Strip Authorization when redirecting to a different host (e.g. S3)
+        if original_uri.host == redirect_uri.host
+          auth = original_request["Authorization"]
+          request["Authorization"] = auth if auth
+        end
+
+        request
+      end
+      private_class_method :build_redirect_request
+    end
+  end
+end

data/lib/braintrust/server/auth/clerk_token.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+
+module Braintrust
+  module Server
+    module Auth
+      # Validates Clerk JWT session tokens via the Braintrust app endpoint.
+      # The browser forwards the Clerk session token which is validated by
+      # POST /api/apikey/login on the app server.
+      class ClerkToken
+        DEFAULT_APP_URL = "https://www.braintrust.dev"
+        RACK_AUTH_HEADER = "HTTP_AUTHORIZATION"
+        RACK_ORG_NAME_HEADER = "HTTP_X_BT_ORG_NAME"
+        BEARER_PATTERN = /\ABearer (.+)\z/
+        LOGIN_PATH = "/api/apikey/login"
+
+        def initialize(app_url: nil)
+          @app_url = app_url || DEFAULT_APP_URL
+        end
+
+        def authenticate(env)
+          token = extract_bearer_token(env)
+          return nil unless token
+
+          login_response = validate_token(token)
+          return nil unless login_response
+
+          org_name = env[RACK_ORG_NAME_HEADER]
+
+          {
+            "api_key" => token,
+            "org_id" => login_response["org_id"],
+            "org_name" => org_name || login_response["org_name"],
+            "app_url" => @app_url,
+            "api_url" => login_response["api_url"] || @app_url
+          }
+        end
+
+        private
+
+        def extract_bearer_token(env)
+          header = env[RACK_AUTH_HEADER]
+          return nil unless header
+          header[BEARER_PATTERN, 1]
+        end
+
+        def validate_token(token)
+          uri = URI("#{@app_url}#{LOGIN_PATH}")
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = uri.scheme == "https"
+
+          request = Net::HTTP::Post.new(uri)
+          request["Content-Type"] = "application/json"
+          request.body = JSON.dump({token: token})
+
+          response = http.request(request)
+          return nil unless response.code == "200"
+
+          JSON.parse(response.body)
+        rescue
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/server/auth/no_auth.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Server
+    module Auth
+      # No-op auth strategy for testing and local development.
+      class NoAuth
+        def authenticate(_env)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/server/handlers/eval.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Braintrust
+  module Server
+    module Handlers
+      # POST /eval — adapter that maps HTTP request to Evaluator#run and streams SSE results.
+      # Handles auth passthrough, datasets, remote scorers, project_id, and parent.
+      class Eval
+        def initialize(evaluators)
+          @evaluators = evaluators
+        end
+
+        def call(env)
+          body = parse_body(env)
+          return error_response(400, "Invalid JSON body") unless body
+
+          name = body["name"]
+          return error_response(400, "Missing required field: name") unless name
+
+          evaluator = @evaluators[name]
+          return error_response(404, "Evaluator '#{name}' not found") unless evaluator
+
+          data = body["data"]
+          return error_response(400, "Missing required field: data") unless data
+
+          # Validate exactly one data source
+          data_sources = ["data", "dataset_name", "dataset_id"].count { |k| data.key?(k) }
+          return error_response(400, "Exactly one data source required") if data_sources != 1
+
+          experiment_name = body["experiment_name"]
+
+          # Resolve data source
+          cases, dataset = resolve_data_source(data)
+
+          # Resolve remote scorers from request
+          remote_scorer_ids = resolve_remote_scorers(body["scores"])
+
+          # Resolve parent span context
+          parent = resolve_parent(body["parent"])
+
+          # Build state from auth context (if present)
+          state = build_state(env)
+
+          # The protocol-rack adapter (used by Falcon and any server built on
+          # protocol-http) buffers `each`-based bodies through an Enumerable path.
+          # Detect it via the "protocol.http.request" env key it injects, and use
+          # SSEStreamBody (call-only) so it dispatches through the Streaming path.
+          body_class = env.key?("protocol.http.request") ? SSEStreamBody : SSEBody
+
+          sse_body = body_class.new do |sse|
+            # Only pass project/experiment params when state is available
+            run_opts = {
+              on_progress: ->(progress_data) {
+                # Build remote eval protocol events from generic progress data.
+                # Runner provides: id, data/error, scores (optional), origin (optional).
+                # Protocol requires: id, object_type, origin, name, format, output_type, event, data.
+                base = {
+                  "object_type" => "task",
+                  "name" => name,
+                  "format" => "code",
+                  "output_type" => "completion"
+                }
+                base["id"] = progress_data["id"] if progress_data["id"]
+                base["origin"] = progress_data["origin"] if progress_data["origin"]
+
+                if progress_data.key?("error")
+                  sse.event("progress", JSON.dump(base.merge("event" => "error", "data" => progress_data["error"])))
+                else
+                  sse.event("progress", JSON.dump(base.merge("event" => "json_delta", "data" => JSON.dump(progress_data["data"]))))
+                end
+
+                # Signal per-cell completion so the UI exits "Streaming..." state
+                # and updates the progress bar immediately.
+                sse.event("progress", JSON.dump(base.merge("event" => "done", "data" => "")))
+              },
+              quiet: true
+            }
+            run_opts[:parent] = parent if parent
+            run_opts[:scorers] = remote_scorer_ids if remote_scorer_ids
+            run_opts[:dataset] = dataset if dataset
+
+            if state
+              run_opts[:state] = state
+              run_opts[:experiment] = experiment_name if experiment_name
+              run_opts[:project_id] = body["project_id"] if body["project_id"]
+            end
+
+            result = evaluator.run(cases, **run_opts)
+
+            # Flush buffered OTLP spans before sending completion events.
+            # The BatchSpanProcessor exports every ~5s; fast evals can finish
+            # before a single export fires, causing the UI to see no results.
+            Braintrust::Trace.flush_spans
+
+            # Build summary from result scores
+            averaged_scores = {}
+            result.scorer_stats.each do |scorer_name, stats|
+              averaged_scores[scorer_name] = stats.score_mean
+            end
+
+            sse.event("summary", JSON.dump({
+              "scores" => averaged_scores,
+              "experiment_name" => experiment_name,
+              "experiment_id" => result.experiment_id,
+              "project_id" => result.project_id
+            }))
+
+            sse.event("done", "")
+          end
+
+          [200, {"content-type" => "text/event-stream", "cache-control" => "no-cache", "connection" => "keep-alive"}, sse_body]
+        end
+
+        private
+
+        # Resolve data source from the data field.
+        # Returns [cases, dataset] where exactly one is non-nil.
+        def resolve_data_source(data)
+          if data.key?("data")
+            cases = data["data"].map do |d|
+              {input: d["input"], expected: d["expected"]}
+            end
+            [cases, nil]
+          elsif data.key?("dataset_id")
+            [nil, Braintrust::DatasetId.new(id: data["dataset_id"])]
+          elsif data.key?("dataset_name")
+            dataset_opts = {name: data["dataset_name"]}
+            dataset_opts[:project] = data["project_name"] if data["project_name"]
+            [nil, dataset_opts]
+          else
+            [nil, nil]
+          end
+        end
+
+        # Map request scores array to ScorerId structs.
+        # The UI sends function_id as a nested object: {"function_id": "uuid"}.
+        def resolve_remote_scorers(scores)
+          return nil if scores.nil? || scores.empty?
+          scores.map do |s|
+            func_id = s["function_id"]
+            func_id = func_id["function_id"] if func_id.is_a?(Hash)
+            Braintrust::ScorerId.new(
+              function_id: func_id,
+              version: s["version"]
+            )
+          end
+        end
+
+        # Map request parent to symbol-keyed Hash.
+        # Hardcode playground_id to match Java SDK behavior.
+        # Also extracts generation from propagated_event for span_attributes.
+        def resolve_parent(parent)
+          return nil unless parent.is_a?(Hash)
+          object_id = parent["object_id"]
+          return nil unless object_id
+
+          generation = parent.dig("propagated_event", "span_attributes", "generation")
+
+          result = {object_type: "playground_id", object_id: object_id}
+          result[:generation] = generation if generation
+          result
+        end
+
+        # Build State from auth context set by Auth middleware.
+        # Returns nil when no auth context is present (e.g. NoAuth strategy).
+        # Uses an LRU-style cache (max 64 entries) keyed by [api_key, app_url, org_name].
+        def build_state(env)
+          auth = env["braintrust.auth"]
+          return nil unless auth.is_a?(Hash)
+
+          cache_key = [auth["api_key"], auth["app_url"], auth["org_name"]]
+
+          @state_mutex ||= Mutex.new
+          @state_cache ||= {}
+
+          @state_mutex.synchronize do
+            cached = @state_cache[cache_key]
+            return cached if cached
+
+            state = Braintrust::State.new(
+              api_key: auth["api_key"],
+              org_id: auth["org_id"],
+              org_name: auth["org_name"],
+              app_url: auth["app_url"],
+              api_url: auth["api_url"],
+              enable_tracing: false
+            )
+
+            # Evict oldest entry if cache is full
+            if @state_cache.size >= 64
+              oldest_key = @state_cache.keys.first
+              @state_cache.delete(oldest_key)
+            end
+
+            @state_cache[cache_key] = state
+            state
+          end
+        end
+
+        def parse_body(env)
+          body = env["rack.input"]&.read
+          return nil if body.nil? || body.empty?
+          JSON.parse(body)
+        rescue JSON::ParserError
+          nil
+        end
+
+        def error_response(status, message)
+          [status, {"content-type" => "application/json"},
+            [JSON.dump({"error" => message})]]
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/server/handlers/health.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Braintrust
+  module Server
+    module Handlers
+      # GET / — simple health check endpoint.
+      class Health
+        def call(_env)
+          [200, {"content-type" => "application/json"}, [JSON.dump({"status" => "ok"})]]
+        end
+      end
+    end
+  end
+end