braintrust 0.1.3 → 0.2.0

data/lib/braintrust/server/handlers/list.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Braintrust
+  module Server
+    module Handlers
+      # GET/POST /list — returns all evaluators keyed by name.
+      #
+      # Response format (Braintrust dev server protocol):
+      #   {
+      #     "evaluator-name": {
+      #       "parameters": {                          # optional
+      #         "type": "braintrust.staticParameters",
+      #         "schema": {
+      #           "param_name": { "type": "data", "schema": {...}, "default": ..., "description": ... }
+      #         },
+      #         "source": null
+      #       },
+      #       "scores": [{ "name": "scorer_name" }, ...]
+      #     }
+      #   }
+      class List
+        def initialize(evaluators)
+          @evaluators = evaluators
+        end
+
+        def call(_env)
+          result = {}
+          @evaluators.each do |name, evaluator|
+            scores = (evaluator.scorers || []).each_with_index.map do |scorer, i|
+              scorer_name = scorer.respond_to?(:name) ? scorer.name : "score_#{i}"
+              {"name" => scorer_name}
+            end
+            entry = {"scores" => scores}
+            params = serialize_parameters(evaluator.parameters)
+            entry["parameters"] = params if params
+            result[name] = entry
+          end
+
+          [200, {"content-type" => "application/json"},
+            [JSON.dump(result)]]
+        end
+
+        private
+
+        # Convert user-defined parameters to the dev server protocol format.
+        # Wraps in a staticParameters container with "data" typed entries.
+        def serialize_parameters(parameters)
+          return nil unless parameters && !parameters.empty?
+
+          schema = {}
+          parameters.each do |name, spec|
+            spec = spec.transform_keys(&:to_s) if spec.is_a?(Hash)
+            if spec.is_a?(Hash)
+              schema[name.to_s] = {
+                "type" => "data",
+                "schema" => {"type" => spec["type"] || "string"},
+                "default" => spec["default"],
+                "description" => spec["description"]
+              }
+            end
+          end
+
+          {
+            "type" => "braintrust.staticParameters",
+            "schema" => schema,
+            "source" => nil
+          }
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/server/middleware/auth.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Braintrust
+  module Server
+    module Middleware
+      # Auth middleware that validates requests using a pluggable strategy.
+      # Sets env["braintrust.auth"] with the authentication result on success.
+      class Auth
+        def initialize(app, strategy:)
+          @app = app
+          @strategy = strategy
+        end
+
+        def call(env)
+          auth_result = @strategy.authenticate(env)
+          unless auth_result
+            return [401, {"content-type" => "application/json"},
+              [JSON.dump({"error" => "Unauthorized"})]]
+          end
+
+          env["braintrust.auth"] = auth_result
+          @app.call(env)
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/server/middleware/cors.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Server
+    module Middleware
+      # CORS middleware allowing requests from *.braintrust.dev origins.
+      # Handles preflight OPTIONS requests and adds CORS headers to all responses.
+      class Cors
+        ALLOWED_ORIGIN_PATTERN = /\Ahttps?:\/\/([\w-]+\.)*braintrust\.dev\z/
+
+        HEADER_ALLOW_ORIGIN = "access-control-allow-origin"
+        HEADER_ALLOW_CREDENTIALS = "access-control-allow-credentials"
+        HEADER_ALLOW_METHODS = "access-control-allow-methods"
+        HEADER_ALLOW_HEADERS = "access-control-allow-headers"
+        HEADER_MAX_AGE = "access-control-max-age"
+        HEADER_ALLOW_PRIVATE_NETWORK = "access-control-allow-private-network"
+        HEADER_EXPOSE_HEADERS = "access-control-expose-headers"
+        EXPOSED_HEADERS = "x-bt-cursor, x-bt-found-existing-experiment, x-bt-span-id, x-bt-span-export"
+
+        ALLOWED_HEADERS = %w[
+          content-type
+          authorization
+          x-amz-date
+          x-api-key
+          x-amz-security-token
+          x-bt-auth-token
+          x-bt-parent
+          x-bt-org-name
+          x-bt-project-id
+          x-bt-stream-fmt
+          x-bt-use-cache
+          x-bt-use-gateway
+          x-stainless-os
+          x-stainless-lang
+          x-stainless-package-version
+          x-stainless-runtime
+          x-stainless-runtime-version
+          x-stainless-arch
+        ].freeze
+
+        def initialize(app)
+          @app = app
+        end
+
+        def call(env)
+          origin = env["HTTP_ORIGIN"]
+
+          if env["REQUEST_METHOD"] == "OPTIONS"
+            return handle_preflight(env, origin)
+          end
+
+          status, headers, body = @app.call(env)
+          add_cors_headers(headers, origin)
+          [status, headers, body]
+        end
+
+        private
+
+        def handle_preflight(env, origin)
+          headers = {}
+          add_cors_headers(headers, origin)
+          headers[HEADER_ALLOW_METHODS] = "GET, POST, OPTIONS"
+          headers[HEADER_ALLOW_HEADERS] = ALLOWED_HEADERS.join(", ")
+          headers[HEADER_MAX_AGE] = "86400"
+
+          if env["HTTP_ACCESS_CONTROL_REQUEST_PRIVATE_NETWORK"] == "true"
+            headers[HEADER_ALLOW_PRIVATE_NETWORK] = "true"
+          end
+
+          [204, headers, []]
+        end
+
+        def add_cors_headers(headers, origin)
+          return unless origin && allowed_origin?(origin)
+
+          headers[HEADER_ALLOW_ORIGIN] = origin
+          headers[HEADER_ALLOW_CREDENTIALS] = "true"
+          headers[HEADER_EXPOSE_HEADERS] = EXPOSED_HEADERS
+        end
+
+        def allowed_origin?(origin)
+          ALLOWED_ORIGIN_PATTERN.match?(origin)
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/server/rack/app.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Server
+    module Rack
+      # Builds the Rack middleware stack for the eval server.
+      class App
+        def self.build(evaluators: {}, auth: :clerk_token)
+          router = Router.new
+          router.add("GET", "/", Handlers::Health.new)
+          list_handler = Handlers::List.new(evaluators)
+          router.add("GET", "/list", list_handler)
+          router.add("POST", "/list", list_handler)
+          router.add("POST", "/eval", Handlers::Eval.new(evaluators))
+
+          auth_strategy = resolve_auth(auth)
+
+          app = router
+          app = Middleware::Auth.new(app, strategy: auth_strategy)
+          Middleware::Cors.new(app)
+        end
+
+        def self.resolve_auth(auth)
+          case auth
+          when :none
+            Auth::NoAuth.new
+          when :clerk_token
+            Auth::ClerkToken.new
+          else
+            auth
+          end
+        end
+
+        private_class_method :resolve_auth
+      end
+    end
+  end
+end

data/lib/braintrust/server/rack.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+begin
+  require "rack"
+rescue LoadError
+  raise LoadError,
+    "The 'rack' gem is required for the Braintrust server. " \
+    "Add `gem 'rack'` to your Gemfile."
+end
+
+require "json"
+require_relative "../eval"
+require_relative "sse"
+require_relative "auth/no_auth"
+require_relative "auth/clerk_token"
+require_relative "middleware/cors"
+require_relative "middleware/auth"
+require_relative "handlers/health"
+require_relative "handlers/list"
+require_relative "handlers/eval"
+require_relative "router"
+require_relative "rack/app"
+
+module Braintrust
+  module Server
+    module Rack
+      # Build the Rack application for the eval server.
+      # @param evaluators [Hash<String, Evaluator>] Named evaluators ({ "name" => instance })
+      # @param auth [:clerk_token, :none, Object] Auth strategy (default: :clerk_token)
+      # @return [#call] Rack application
+      def self.app(evaluators: {}, auth: :clerk_token)
+        App.build(evaluators: evaluators, auth: auth)
+      end
+    end
+  end
+end

data/lib/braintrust/server/router.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Braintrust
+  module Server
+    # Simple request router that dispatches to handlers based on method + path.
+    # Returns 405 for known paths with wrong method, 404 for unknown paths.
+    class Router
+      def initialize
+        @routes = {}
+      end
+
+      def add(method, path, handler)
+        @routes["#{method} #{path}"] = handler
+        self
+      end
+
+      def call(env)
+        method = env["REQUEST_METHOD"]
+        path = env["PATH_INFO"]
+
+        handler = @routes["#{method} #{path}"]
+        return handler.call(env) if handler
+
+        # Path exists but wrong method
+        if @routes.any? { |key, _| key.end_with?(" #{path}") }
+          return [405, {"content-type" => "application/json"},
+            [JSON.dump({"error" => "Method not allowed"})]]
+        end
+
+        [404, {"content-type" => "application/json"},
+          [JSON.dump({"error" => "Not found"})]]
+      end
+    end
+  end
+end

data/lib/braintrust/server/sse.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Server
+    # Rack-compatible response body that streams SSE events via `each`.
+    #
+    # Works with Puma (immediate writes), Passenger, and rack-test.
+    # WEBrick buffers the entire body and is unsuitable for SSE.
+    #
+    # Falcon buffers `each`-based bodies as Enumerable; use SSEStreamBody instead.
+    class SSEBody
+      def initialize(&block)
+        @block = block
+      end
+
+      def each
+        writer = SSEWriter.new { |chunk| yield chunk }
+        @block.call(writer)
+      end
+    end
+
+    # Rack 3 streaming response body that writes SSE events via `call(stream)`.
+    #
+    # Required for servers using the protocol-rack adapter (e.g. Falcon), which
+    # dispatches `each`-based bodies through a buffered Enumerable path. Bodies
+    # that respond only to `call` are dispatched through the Streaming path for
+    # true async writes.
+    class SSEStreamBody
+      def initialize(&block)
+        @block = block
+      end
+
+      def call(stream)
+        writer = SSEWriter.new { |chunk| stream.write(chunk) }
+        @block.call(writer)
+      ensure
+        stream.close
+      end
+    end
+
+    # Writes formatted SSE events.
+    class SSEWriter
+      def initialize(&block)
+        @write = block
+      end
+
+      def event(type, data = "")
+        @write.call("event: #{type}\ndata: #{data}\n\n")
+      end
+    end
+  end
+end

data/lib/braintrust/server.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require_relative "server/rack"
+
+module Braintrust
+  module Server
+  end
+end

data/lib/braintrust/trace/attachment.rb

@@ -2,6 +2,7 @@
 
 require "net/http"
 require_relative "../internal/encoding"
+require_relative "../internal/http"
 require "uri"
 
 module Braintrust
@@ -91,7 +92,8 @@ module Braintrust
       #   att = Braintrust::Trace::Attachment.from_url("https://example.com/image.png")
       def self.from_url(url)
         uri = URI.parse(url)
-        response = Net::HTTP.get_response(uri)
+        request = Net::HTTP::Get.new(uri)
+        response = Braintrust::Internal::Http.with_redirects(uri, request)
 
         unless response.is_a?(Net::HTTPSuccess)
           raise StandardError, "Failed to fetch URL: #{response.code} #{response.message}"

data/lib/braintrust/trace/span_exporter.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "opentelemetry/exporter/otlp"
+
+module Braintrust
+  module Trace
+    # Custom OTLP exporter that groups spans by braintrust.parent attribute
+    # and sets the x-bt-parent HTTP header per group. This is required for
+    # the Braintrust OTLP backend to route spans to the correct experiment/project.
+    #
+    # Thread safety: BatchSpanProcessor serializes export() calls via its
+    # @export_mutex, so @headers mutation here is safe.
+    class SpanExporter < OpenTelemetry::Exporter::OTLP::Exporter
+      PARENT_ATTR_KEY = SpanProcessor::PARENT_ATTR_KEY
+      PARENT_HEADER = "x-bt-parent"
+
+      SUCCESS = OpenTelemetry::SDK::Trace::Export::SUCCESS
+      FAILURE = OpenTelemetry::SDK::Trace::Export::FAILURE
+
+      def initialize(endpoint:, api_key:)
+        super(endpoint: endpoint, headers: {"Authorization" => "Bearer #{api_key}"})
+      end
+
+      def export(span_data, timeout: nil)
+        failed = false
+        span_data.group_by { |sd| sd.attributes&.[](PARENT_ATTR_KEY) }.each do |parent_value, spans|
+          @headers[PARENT_HEADER] = parent_value if parent_value
+          failed = true unless super(spans, timeout: timeout) == SUCCESS
+        ensure
+          @headers.delete(PARENT_HEADER)
+        end
+        failed ? FAILURE : SUCCESS
+      end
+    end
+  end
+end

data/lib/braintrust/trace.rb

@@ -3,6 +3,7 @@
 require "opentelemetry/sdk"
 require "opentelemetry/exporter/otlp"
 require_relative "trace/span_processor"
+require_relative "trace/span_exporter"
 require_relative "trace/span_filter"
 require_relative "internal/env"
 require_relative "logger"
@@ -88,11 +89,9 @@ module Braintrust
       config ||= state.respond_to?(:config) ? state.config : nil
 
       # Create OTLP HTTP exporter unless override provided
-      exporter ||= OpenTelemetry::Exporter::OTLP::Exporter.new(
+      exporter ||= SpanExporter.new(
         endpoint: "#{state.api_url}/otel/v1/traces",
-        headers: {
-          "Authorization" => "Bearer #{state.api_key}"
-        }
+        api_key: state.api_key
       )
 
       # Use SimpleSpanProcessor for InMemorySpanExporter (testing), BatchSpanProcessor for production

data/lib/braintrust/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Braintrust
-  VERSION = "0.1.3"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: braintrust
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Braintrust
@@ -234,6 +234,7 @@ files:
 - lib/braintrust/eval.rb
 - lib/braintrust/eval/case.rb
 - lib/braintrust/eval/cases.rb
+- lib/braintrust/eval/evaluator.rb
 - lib/braintrust/eval/formatter.rb
 - lib/braintrust/eval/functions.rb
 - lib/braintrust/eval/result.rb
@@ -242,16 +243,30 @@ files:
 - lib/braintrust/eval/summary.rb
 - lib/braintrust/internal/encoding.rb
 - lib/braintrust/internal/env.rb
+- lib/braintrust/internal/http.rb
 - lib/braintrust/internal/origin.rb
 - lib/braintrust/internal/template.rb
 - lib/braintrust/internal/thread_pool.rb
 - lib/braintrust/internal/time.rb
 - lib/braintrust/logger.rb
 - lib/braintrust/prompt.rb
+- lib/braintrust/server.rb
+- lib/braintrust/server/auth/clerk_token.rb
+- lib/braintrust/server/auth/no_auth.rb
+- lib/braintrust/server/handlers/eval.rb
+- lib/braintrust/server/handlers/health.rb
+- lib/braintrust/server/handlers/list.rb
+- lib/braintrust/server/middleware/auth.rb
+- lib/braintrust/server/middleware/cors.rb
+- lib/braintrust/server/rack.rb
+- lib/braintrust/server/rack/app.rb
+- lib/braintrust/server/router.rb
+- lib/braintrust/server/sse.rb
 - lib/braintrust/setup.rb
 - lib/braintrust/state.rb
 - lib/braintrust/trace.rb
 - lib/braintrust/trace/attachment.rb
+- lib/braintrust/trace/span_exporter.rb
 - lib/braintrust/trace/span_filter.rb
 - lib/braintrust/trace/span_processor.rb
 - lib/braintrust/vendor/mustache.rb