braintrust 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d67e6d0faeb24297af8a5f43ac1bd1ceacff1f37df2610244ae5f81e34c4ae5f
-  data.tar.gz: 489ec68fee424aa8aa1880b73b58f1f26529493d8898cd0ae5876d3b919fcb7c
+  metadata.gz: c07be3c454a924c5c97c2653136a2b9cdd1098409af16326b1db8676c5c8b0d2
+  data.tar.gz: c1eb75eefdcacebc2c955ae23aa3196d276a76d6ab828cdfb817c7e9168325b3
 SHA512:
-  metadata.gz: cd876122ad92c5439ff45e975fd84418bfcc7d72d6f9398e48b1ac4c60f09fb96c2b85b46ee1c8de6a75291c0b7d2754ee2fa069f77f8a2f8a4c069132c59d94
-  data.tar.gz: 45d3f80f69ac9725d93aa0db24815da093bfd992b5418f8551c8d25e8caef9299f270a92fa922a4bc4bf3190d9f823a35c7203f9a74bd58daee31869b987f103
+  metadata.gz: d02058bd5321ed16ea2f785aaeb24f4d4f105c5357c3c7ceb2a8a02c090b69c7187623b23e14d5026bb0cf236e64dddae7025509d7b2d6769bb50f110612120f
+  data.tar.gz: 15627209b382c023c2640e1d2219b6d33b84cb7c67ba1a3b8e3ebbe1aa912d3df832583a1e37b3831699b67ea81f3b4242b67a606dfdd727827e648a6509fea7

data/README.md

@@ -252,13 +252,15 @@ Braintrust::Eval.run(
     {input: "apple", expected: "fruit"},
     {input: "carrot", expected: "vegetable"}
   ],
-  task: ->(input) { classify(input) },
+  task: ->(input:) { classify(input) },
   scorers: [
-    ->(input, expected, output) { output == expected ? 1.0 : 0.0 }
+    ->(expected:, output:) { output == expected ? 1.0 : 0.0 }
   ]
 )
 ```
 
+See [eval.rb](./examples/eval.rb) for a full example.
+
 ### Datasets
 
 Use test cases from a Braintrust dataset:
@@ -267,7 +269,7 @@ Use test cases from a Braintrust dataset:
 Braintrust::Eval.run(
   project: "my-project",
   dataset: "my-dataset",
-  task: ->(input) { classify(input) },
+  task: ->(input:) { classify(input) },
   scorers: [...]
 )
 ```
@@ -282,11 +284,13 @@ Braintrust::Eval.run(
     {input: "apple", expected: "fruit", tags: ["produce"], metadata: {difficulty: "easy"}},
     {input: "salmon", expected: "protein", tags: ["seafood"], metadata: {difficulty: "medium"}}
   ],
-  task: ->(input) { classify(input) },
+  task: ->(input:) { classify(input) },
   scorers: [...]
 )
 ```
 
+See [dataset.rb](./examples/eval/dataset.rb) for a full example.
+
 ### Scorers
 
 Use scoring functions defined in Braintrust:
@@ -295,33 +299,104 @@ Use scoring functions defined in Braintrust:
 Braintrust::Eval.run(
   project: "my-project",
   cases: [...],
-  task: ->(input) { ... },
+  task: ->(input:) { ... },
+  scorers: ["accuracy-scorer"]
+)
+```
+
+Or define scorers inline with `Scorer.new`:
+
+```ruby
+Braintrust::Eval.run(
+  project: "my-project",
+  cases: [...],
+  task: ->(input:) { ... },
   scorers: [
-    Braintrust::Eval::Functions.scorer(project: "my-project", slug: "accuracy-scorer")
+    Braintrust::Scorer.new("exact_match") do |expected:, output:|
+      output == expected ? 1.0 : 0.0
+    end
   ]
 )
 ```
 
-Or define scorers inline with `Eval.scorer`:
+See [remote_functions.rb](./examples/eval/remote_functions.rb) for a full example.
+
+#### Scorer metadata
+
+Scorers can return a Hash with `:score` and `:metadata` to attach structured context to the score. The metadata is logged on the scorer's span and visible in the Braintrust UI for debugging and filtering:
+
+```ruby
+Braintrust::Scorer.new("translation") do |expected:, output:|
+  common_words = output.downcase.split & expected.downcase.split
+  overlap = common_words.size.to_f / expected.split.size
+  {
+    score: overlap,
+    metadata: {word_overlap: common_words.size, missing_words: expected.downcase.split - output.downcase.split}
+  }
+end
+```
+
+See [scorer_metadata.rb](./examples/eval/scorer_metadata.rb) for a full example.
+
+#### Multiple scores from one scorer
+
+When several scores can be computed together (e.g. in one LLM call), you can return an `Array` of score `Hash` instead of a single value. Each metric appears as a separate score column in the Braintrust UI:
+
+```ruby
+Braintrust::Scorer.new("summary_quality") do |output:, expected:|
+  words = output.downcase.split
+  key_terms = expected[:key_terms]
+  covered = key_terms.count { |t| words.include?(t) }
+
+  [
+    {name: "coverage", score: covered.to_f / key_terms.size, metadata: {missing: key_terms - words}},
+    {name: "conciseness", score: words.size <= expected[:max_words] ? 1.0 : 0.0}
+  ]
+end
+```
+
+`name` and `score` are required, `metadata` is optional.
+
+See [multi_score.rb](./examples/eval/multi_score.rb) for a full example.
+
+#### Trace scoring
+
+Scorers can access the full evaluation trace (all spans generated by the task) by declaring a `trace:` keyword parameter. This is useful for inspecting intermediate LLM calls, validating tool usage, or checking the message thread:
 
 ```ruby
 Braintrust::Eval.run(
   project: "my-project",
-  cases: [...],
-  task: ->(input) { ... },
+  cases: [{input: "What is 2+2?", expected: "4"}],
+  task: Braintrust::Task.new { |input:| my_llm_pipeline(input) },
   scorers: [
-    Braintrust::Eval.scorer("exact_match") do |input, expected, output|
+    # Access the full trace to inspect LLM spans
+    Braintrust::Scorer.new("uses_system_prompt") do |output:, trace:|
+      messages = trace.thread  # reconstructed message thread from LLM spans
+      messages.any? { |m| m["role"] == "system" } ? 1.0 : 0.0
+    end,
+
+    # Filter spans by type
+    Braintrust::Scorer.new("single_llm_call") do |output:, trace:|
+      trace.spans(span_type: "llm").length == 1 ? 1.0 : 0.0
+    end,
+
+    # Scorers without trace: still work — the parameter is filtered out automatically
+    Braintrust::Scorer.new("exact_match") do |output:, expected:|
       output == expected ? 1.0 : 0.0
     end
   ]
 )
 ```
 
-See examples: [eval.rb](./examples/eval.rb), [dataset.rb](./examples/eval/dataset.rb), [remote_functions.rb](./examples/eval/remote_functions.rb)
+See [trace_scoring.rb](./examples/eval/trace_scoring.rb) for a full example.
 
 ### Dev Server
 
-Run evaluations from the Braintrust web UI against code in your own application. Define evaluators, pass them to the dev server, and start serving:
+Run evaluations from the Braintrust web UI against code in your own application.
+
+#### Run as a Rack app
+
+Define evaluators, pass them to the dev server, and start serving:
 
 ```ruby
 # eval_server.ru
@@ -330,9 +405,9 @@ require "braintrust/server"
 
 # Define evaluators — these can reference your application code (models, services, etc.)
 food_classifier = Braintrust::Eval::Evaluator.new(
-  task: ->(input) { FoodClassifier.classify(input) },
+  task: ->(input:) { FoodClassifier.classify(input) },
   scorers: [
-    Braintrust::Eval.scorer("exact_match") { |input, expected, output| output == expected ? 1.0 : 0.0 }
+    Braintrust::Scorer.new("exact_match") { |expected:, output:| output == expected ? 1.0 : 0.0 }
   ]
 )
 
@@ -347,10 +422,21 @@ run Braintrust::Server::Rack.app(
 )
 ```
 
+Add your Rack server to your Gemfile:
+
+```ruby
+gem "rack"
+gem "puma" # recommended
+```
+
+Then start the server:
+
 ```bash
 bundle exec rackup eval_server.ru -p 8300 -o 0.0.0.0
 ```
 
+See example: [server/eval.ru](./examples/server/eval.ru)
+
 **Custom evaluators**
 
 Evaluators can also be defined as subclasses:
@@ -358,15 +444,60 @@ Evaluators can also be defined as subclasses:
 ```ruby
 class FoodClassifier < Braintrust::Eval::Evaluator
   def task
-    ->(input) { classify(input) }
+    ->(input:) { classify(input) }
+  end
+
+  def scorers
+    [Braintrust::Scorer.new("exact_match") { |expected:, output:| output == expected ? 1.0 : 0.0 }]
+  end
+end
+```
+
+#### Run as a Rails engine
+
+Use the Rails engine when your evaluators live inside an existing Rails app and you want to mount the Braintrust eval server into that application.
+
+Define each evaluator in its own file, for example under `app/evaluators/`:
+
+```ruby
+# app/evaluators/food_classifier.rb
+class FoodClassifier < Braintrust::Eval::Evaluator
+  def task
+    ->(input:) { classify(input) }
   end
 
   def scorers
-    [Braintrust::Eval.scorer("exact_match") { |i, e, o| o == e ? 1.0 : 0.0 }]
+    [Braintrust::Scorer.new("exact_match") { |expected:, output:| output == expected ? 1.0 : 0.0 }]
   end
 end
 ```
 
+Then generate the Braintrust initializer:
+
+```bash
+bin/rails generate braintrust:eval_server
+```
+
+```ruby
+# config/routes.rb
+Rails.application.routes.draw do
+  mount Braintrust::Contrib::Rails::Engine, at: "/braintrust"
+end
+```
+
+The generator writes `config/initializers/braintrust_server.rb`, where you can review or customize the slug-to-evaluator mapping it discovers from `app/evaluators/**/*.rb` and `evaluators/**/*.rb`.
+
+See example: [contrib/rails/eval.rb](./examples/contrib/rails/eval.rb)
+
+**Developing locally**
+
+If you want to skip authentication on incoming eval requests while developing locally:
+
+- **For Rack**: Pass `auth: :none` to `Braintrust::Server::Rack.app(...)`
+- **For Rails**: Set `config.auth = :none` in `config/initializers/braintrust_server.rb`
+
+*NOTE: Setting `:none` disables authentication on incoming requests into your server; executing evals requires a `BRAINTRUST_API_KEY` to fetch resources.*
+
 **Supported web servers**
 
 The dev server requires the `rack` gem and a Rack-compatible web server.
@@ -378,14 +509,7 @@ The dev server requires the `rack` gem and a Rack-compatible web server.
 | [Passenger](https://www.phusionpassenger.com/) | 6.x               |                                      |
 | [WEBrick](https://github.com/ruby/webrick)     | Not supported     | Does not support server-sent events. |
 
-Add your chosen server to your Gemfile:
-
-```ruby
-gem "rack"
-gem "puma" # recommended
-```
-
-See example: [server/eval.ru](./examples/server/eval.ru)
+See examples: [server/eval.ru](./examples/server/eval.ru), 
 
 ## Documentation
 

data/lib/braintrust/api/internal/btql.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "uri"
+require_relative "../../internal/http"
+
+module Braintrust
+  class API
+    module Internal
+      # Internal BTQL client for querying spans.
+      # Not part of the public API — instantiated directly where needed.
+      class BTQL
+        # Maximum number of retries before returning partial results.
+        # Covers both freshness lag (partially indexed) and ingestion lag
+        # (spans not yet visible to BTQL after OTel flush).
+        MAX_FRESHNESS_RETRIES = 7
+
+        # Base delay (seconds) between retries (doubles each attempt, capped).
+        FRESHNESS_BASE_DELAY = 1.0
+
+        # Maximum delay (seconds) between retries. Caps exponential growth
+        # so we keep polling at a reasonable rate in the later window.
+        # Schedule: 1, 2, 4, 8, 8, 8, 8 = ~39s total worst-case.
+        MAX_FRESHNESS_DELAY = 8.0
+
+        def initialize(state)
+          @state = state
+        end
+
+        # Query spans belonging to a specific trace within an object.
+        #
+        # Builds a BTQL SQL query that matches the root_span_id and excludes scorer spans.
+        # Retries with exponential backoff if the response indicates data is not yet fresh.
+        #
+        # @param object_type [String] e.g. "experiment"
+        # @param object_id [String] Object UUID
+        # @param root_span_id [String] Hex trace ID of the root span
+        # @return [Array<Hash>] Parsed span data
+        def trace_spans(object_type:, object_id:, root_span_id:)
+          query = build_trace_query(
+            object_type: object_type,
+            object_id: object_id,
+            root_span_id: root_span_id
+          )
+          payload = {query: query, fmt: "jsonl"}
+
+          retries = 0
+          loop do
+            rows, freshness = execute_query(payload)
+            # Return when data is fresh AND non-empty, or we've exhausted retries.
+            # We retry on empty even when "complete" because there is ingestion lag
+            # between OTel flush and BTQL indexing — the server may report "complete"
+            # before it knows about newly-flushed spans.
+            return rows if (freshness == "complete" && !rows.empty?) || retries >= MAX_FRESHNESS_RETRIES
+
+            retries += 1
+            delay = [FRESHNESS_BASE_DELAY * (2**(retries - 1)), MAX_FRESHNESS_DELAY].min
+            sleep(delay)
+          end
+        rescue => e
+          Braintrust::Log.warn("[BTQL] Query failed: #{e.message}")
+          []
+        end
+
+        private
+
+        # Build a BTQL SQL query string for fetching trace spans.
+        #
+        # Selects all spans for a given root_span_id, excluding scorer spans
+        # (span_attributes.type = 'score').
+        #
+        # @param object_type [String] e.g. "experiment"
+        # @param object_id [String] Object UUID
+        # @param root_span_id [String] Hex trace ID
+        # @return [String] BTQL SQL query
+        def build_trace_query(object_type:, object_id:, root_span_id:)
+          escaped_root = root_span_id.gsub("'", "''")
+          escaped_id = object_id.gsub("'", "''")
+
+          "SELECT * FROM #{object_type}('#{escaped_id}') " \
+            "WHERE root_span_id = '#{escaped_root}' " \
+            "AND span_attributes.type != 'score' " \
+            "LIMIT 1000"
+        end
+
+        # Execute a BTQL query and parse the JSONL response.
+        #
+        # @param payload [Hash] BTQL request payload
+        # @return [Array(Array<Hash>, String)] [parsed_rows, freshness_state]
+        def execute_query(payload)
+          uri = URI("#{@state.api_url}/btql")
+
+          request = Net::HTTP::Post.new(uri)
+          request["Content-Type"] = "application/json"
+          request["Authorization"] = "Bearer #{@state.api_key}"
+          request["Accept"] = "application/x-jsonlines"
+          request.body = JSON.dump(payload)
+
+          response = Braintrust::Internal::Http.with_redirects(uri, request)
+
+          unless response.is_a?(Net::HTTPSuccess)
+            raise Braintrust::Error, "HTTP #{response.code} for POST #{uri}: #{response.body}"
+          end
+
+          freshness = response["x-bt-freshness-state"] || "complete"
+          [parse_jsonl(response.body), freshness]
+        end
+
+        # Parse a JSONL response body into an array of hashes.
+        #
+        # @param body [String] JSONL response body
+        # @return [Array<Hash>]
+        def parse_jsonl(body)
+          body.each_line.filter_map do |line|
+            line = line.strip
+            next if line.empty?
+            JSON.parse(line)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/api/internal/experiments.rb

@@ -50,6 +50,25 @@ module Braintrust
 
           JSON.parse(response.body)
         end
+
+        # Delete an experiment
+        # DELETE /v1/experiment/:id
+        # @param id [String] Experiment ID
+        # @return [Hash] Deleted experiment data
+        def delete(id:)
+          uri = URI("#{@state.api_url}/v1/experiment/#{id}")
+
+          request = Net::HTTP::Delete.new(uri)
+          request["Authorization"] = "Bearer #{@state.api_key}"
+
+          response = Braintrust::Internal::Http.with_redirects(uri, request)
+
+          unless response.is_a?(Net::HTTPSuccess)
+            raise Error, "HTTP #{response.code} for DELETE #{uri}: #{response.body}"
+          end
+
+          JSON.parse(response.body)
+        end
       end
     end
   end

data/lib/braintrust/api/internal/projects.rb

@@ -35,6 +35,25 @@ module Braintrust
 
           JSON.parse(response.body)
         end
+
+        # Delete a project
+        # DELETE /v1/project/:id
+        # @param id [String] Project UUID
+        # @return [Hash] Deleted project data
+        def delete(id:)
+          uri = URI("#{@state.api_url}/v1/project/#{id}")
+
+          request = Net::HTTP::Delete.new(uri)
+          request["Authorization"] = "Bearer #{@state.api_key}"
+
+          response = Braintrust::Internal::Http.with_redirects(uri, request)
+
+          unless response.is_a?(Net::HTTPSuccess)
+            raise Error, "HTTP #{response.code} for DELETE #{uri}: #{response.body}"
+          end
+
+          JSON.parse(response.body)
+        end
       end
     end
   end

data/lib/braintrust/contrib/rails/server/application_controller.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Contrib
+    module Rails
+      module Server
+        class ApplicationController < ActionController::API
+          before_action :authenticate!
+
+          private
+
+          def authenticate!
+            auth_result = Engine.auth_strategy.authenticate(request.env)
+            unless auth_result
+              render json: {"error" => "Unauthorized"}, status: :unauthorized
+              return
+            end
+
+            request.env["braintrust.auth"] = auth_result
+            @braintrust_auth = auth_result
+          end
+
+          def parse_json_body
+            body = request.body.read
+            return nil if body.nil? || body.empty?
+            JSON.parse(body)
+          rescue JSON::ParserError
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/contrib/rails/server/engine.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Contrib
+    module Rails
+      module Server
+        class Engine < ::Rails::Engine
+          isolate_namespace Braintrust::Contrib::Rails::Server
+
+          config.evaluators = {}
+          config.auth = :clerk_token
+
+          # Register the engine's routes file so Rails loads it during initialization.
+          paths["config/routes.rb"] << File.expand_path("routes.rb", __dir__)
+
+          initializer "braintrust.server.cors" do |app|
+            app.middleware.use Braintrust::Server::Middleware::Cors
+          end
+
+          # Class-level helpers that read from engine config.
+
+          def self.evaluators
+            config.evaluators
+          end
+
+          def self.auth_strategy
+            resolve_auth(config.auth)
+          end
+
+          def self.list_service
+            Braintrust::Server::Services::List.new(-> { config.evaluators })
+          end
+
+          # Long-lived so the state cache persists across requests.
+          def self.eval_service
+            @eval_service ||= Braintrust::Server::Services::Eval.new(-> { config.evaluators })
+          end
+
+          # Support the explicit `|config|` style used by this integration while
+          # still delegating zero-arity DSL blocks to Rails' native implementation.
+          def self.configure(&block)
+            return super if block&.arity == 0
+            yield config if block
+          end
+
+          def self.resolve_auth(auth)
+            case auth
+            when :none
+              Braintrust::Server::Auth::NoAuth.new
+            when :clerk_token
+              Braintrust::Server::Auth::ClerkToken.new
+            when Symbol, String
+              raise ArgumentError, "Unknown auth strategy #{auth.inspect}. Expected :none, :clerk_token, or an auth object."
+            else
+              auth
+            end
+          end
+          private_class_method :resolve_auth
+
+          generators do
+            require "braintrust/contrib/rails/server/generator"
+          end
+        end
+      end
+    end
+  end
+end
+
+require_relative "application_controller"
+require_relative "health_controller"
+require_relative "list_controller"
+require_relative "eval_controller"

data/lib/braintrust/contrib/rails/server/eval_controller.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Contrib
+    module Rails
+      module Server
+        class EvalController < ApplicationController
+          include ActionController::Live
+
+          def create
+            body = parse_json_body
+            unless body
+              render json: {"error" => "Invalid JSON body"}, status: :bad_request
+              return
+            end
+
+            result = Engine.eval_service.validate(body)
+            if result[:error]
+              render json: {"error" => result[:error]}, status: result[:status]
+              return
+            end
+
+            response.headers["Content-Type"] = "text/event-stream"
+            response.headers["Cache-Control"] = "no-cache"
+            response.headers["Connection"] = "keep-alive"
+
+            sse = Braintrust::Server::SSEWriter.new { |chunk| response.stream.write(chunk) }
+            Engine.eval_service.stream(result, auth: @braintrust_auth, sse: sse)
+          ensure
+            response.stream.close
+          end
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/contrib/rails/server/generator.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module Braintrust
+  module Contrib
+    module Rails
+      module Server
+        module Generators
+          class ServerGenerator < ::Rails::Generators::Base
+            namespace "braintrust:server"
+            source_root File.expand_path("templates", __dir__)
+
+            def create_initializer
+              @evaluators = discovered_evaluators
+              template "initializer.rb.tt", "config/initializers/braintrust_server.rb"
+            end
+
+            private
+
+            def discovered_evaluators
+              evaluator_roots.flat_map do |root|
+                Dir[File.join(destination_root, root, "**/*.rb")].sort.map do |file|
+                  relative_path = file.delete_prefix("#{File.join(destination_root, root)}/").sub(/\.rb\z/, "")
+                  {
+                    class_name: relative_path.split("/").map(&:camelize).join("::"),
+                    slug: relative_path.tr("/", "-").tr("_", "-")
+                  }
+                end
+              end
+            end
+
+            def evaluator_roots
+              %w[app/evaluators evaluators].select do |root|
+                Dir.exist?(File.join(destination_root, root))
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/contrib/rails/server/health_controller.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Contrib
+    module Rails
+      module Server
+        class HealthController < ApplicationController
+          def show
+            render json: {"status" => "ok"}
+          end
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/contrib/rails/server/list_controller.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Contrib
+    module Rails
+      module Server
+        class ListController < ApplicationController
+          def show
+            result = Engine.list_service.call
+            render json: result
+          end
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/contrib/rails/server/routes.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+Braintrust::Contrib::Rails::Server::Engine.routes.draw do
+  get "/", to: "health#show"
+  get "/list", to: "list#show"
+  post "/list", to: "list#show"
+  post "/eval", to: "eval#create"
+end