braintrust 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c07be3c454a924c5c97c2653136a2b9cdd1098409af16326b1db8676c5c8b0d2
-  data.tar.gz: c1eb75eefdcacebc2c955ae23aa3196d276a76d6ab828cdfb817c7e9168325b3
+  metadata.gz: 27e146b06451b844b1e6416353b20f6bd572c3d1169a12a439745cb7280ce0ec
+  data.tar.gz: d726e3a146a2180bf2714846d56e65fa9d3ef1ce773adb116a8e6b1b79ba823c
 SHA512:
-  metadata.gz: d02058bd5321ed16ea2f785aaeb24f4d4f105c5357c3c7ceb2a8a02c090b69c7187623b23e14d5026bb0cf236e64dddae7025509d7b2d6769bb50f110612120f
-  data.tar.gz: 15627209b382c023c2640e1d2219b6d33b84cb7c67ba1a3b8e3ebbe1aa912d3df832583a1e37b3831699b67ea81f3b4242b67a606dfdd727827e648a6509fea7
+  metadata.gz: 69e5150452e9dde1491664af1137cc05a9a5b651dbb5fdee27ff8a09e0e11b51c283c163019566045e1771679ed6f2eece4dd1753aa06f899e3681e7c6b99d15
+  data.tar.gz: 28cc8c86bdc13db8d33ad0dc28325c0d858f37ba1b9f41212c52e514eed649b14596c66153bca58de251c4c6dd1ddcb170d24ae100a33f912f49349671821f7a

data/README.md

@@ -21,6 +21,7 @@ This is the official Ruby SDK for [Braintrust](https://www.braintrust.dev), for
   - [Attachments](#attachments)
   - [Viewing traces](#viewing-traces)
 - [Evals](#evals)
+  - [Tasks](#tasks)
   - [Datasets](#datasets)
   - [Scorers](#scorers)
   - [Dev Server](#dev-server)
@@ -261,6 +262,48 @@ Braintrust::Eval.run(
 
 See [eval.rb](./examples/eval.rb) for a full example.
 
+### Tasks
+
+Define the code being evaluated as a lambda or a class. Tasks receive `input:` as a keyword argument:
+
+```ruby
+# Lambda
+task = ->(input:) { classify(input) }
+
+# Class-based (auto-derives name from class: "food_classifier")
+class FoodClassifier
+  include Braintrust::Task
+
+  def call(input:)
+    classify(input)
+  end
+end
+```
+
+#### With parameters
+
+Tasks can accept `parameters:` as input to drive their behavior:
+
+```ruby
+task = ->(input:, parameters:) {
+  value = parameters["value"]
+  from_unit = parameters["to_unit"] || 'c'
+  to_unit = parameters["from_unit"] || 'f'
+
+  convert_temp(temperature: value, from_unit: from_unit , to_unit: to_unit)
+}
+
+Braintrust::Eval.run(
+  project: "my-project",
+  cases: [...],
+  task: task,
+  scorers: [...],
+  parameters: {"value" => 23.0}
+)
+```
+
+See [parameters.rb](./examples/eval/parameters.rb) for a full example.
+
 ### Datasets
 
 Use test cases from a Braintrust dataset:
@@ -390,6 +433,19 @@ Braintrust::Eval.run(
 
 See [trace_scoring.rb](./examples/eval/trace_scoring.rb) for a full example.
 
+#### Scorer parameters
+
+Scorers can also accept `parameters:` to use runtime configuration in their scoring logic. Like tasks, scorers that don't declare `parameters:` are unaffected:
+
+```ruby
+Braintrust::Scorer.new("threshold_match") do |expected:, output:, parameters:|
+  threshold = parameters["threshold"] || 0.8
+  similarity(output, expected) >= threshold ? 1.0 : 0.0
+end
+```
+
+See [parameters.rb](./examples/eval/parameters.rb) for a full example.
+
 ### Dev Server
 
 Run evaluations from the Braintrust web UI against code in your own application.

data/lib/braintrust/api/functions.rb

@@ -25,13 +25,15 @@ module Braintrust
       # List functions with optional filters
       # GET /v1/function?project_name=X&...
       # @param project_name [String, nil] Filter by project name
+      # @param project_id [String, nil] Filter by project ID (UUID)
       # @param function_name [String, nil] Filter by function name
       # @param slug [String, nil] Filter by slug
       # @param limit [Integer, nil] Limit number of results
       # @return [Hash] Response with "objects" array
-      def list(project_name: nil, function_name: nil, slug: nil, limit: nil)
+      def list(project_name: nil, project_id: nil, function_name: nil, slug: nil, limit: nil)
         params = {}
         params["project_name"] = project_name if project_name
+        params["project_id"] = project_id if project_id
         params["function_name"] = function_name if function_name
         params["slug"] = slug if slug
         params["limit"] = limit if limit

data/lib/braintrust/api/internal/btql.rb

@@ -11,19 +11,6 @@ module Braintrust
       # Internal BTQL client for querying spans.
       # Not part of the public API — instantiated directly where needed.
       class BTQL
-        # Maximum number of retries before returning partial results.
-        # Covers both freshness lag (partially indexed) and ingestion lag
-        # (spans not yet visible to BTQL after OTel flush).
-        MAX_FRESHNESS_RETRIES = 7
-
-        # Base delay (seconds) between retries (doubles each attempt, capped).
-        FRESHNESS_BASE_DELAY = 1.0
-
-        # Maximum delay (seconds) between retries. Caps exponential growth
-        # so we keep polling at a reasonable rate in the later window.
-        # Schedule: 1, 2, 4, 8, 8, 8, 8 = ~39s total worst-case.
-        MAX_FRESHNESS_DELAY = 8.0
-
         def initialize(state)
           @state = state
         end
@@ -31,36 +18,19 @@ module Braintrust
         # Query spans belonging to a specific trace within an object.
         #
         # Builds a BTQL SQL query that matches the root_span_id and excludes scorer spans.
-        # Retries with exponential backoff if the response indicates data is not yet fresh.
+        # Returns a single-shot result; callers are responsible for retry and error handling.
         #
         # @param object_type [String] e.g. "experiment"
         # @param object_id [String] Object UUID
         # @param root_span_id [String] Hex trace ID of the root span
-        # @return [Array<Hash>] Parsed span data
+        # @return [Array(Array<Hash>, String)] [rows, freshness]
         def trace_spans(object_type:, object_id:, root_span_id:)
           query = build_trace_query(
             object_type: object_type,
             object_id: object_id,
             root_span_id: root_span_id
           )
-          payload = {query: query, fmt: "jsonl"}
-
-          retries = 0
-          loop do
-            rows, freshness = execute_query(payload)
-            # Return when data is fresh AND non-empty, or we've exhausted retries.
-            # We retry on empty even when "complete" because there is ingestion lag
-            # between OTel flush and BTQL indexing — the server may report "complete"
-            # before it knows about newly-flushed spans.
-            return rows if (freshness == "complete" && !rows.empty?) || retries >= MAX_FRESHNESS_RETRIES
-
-            retries += 1
-            delay = [FRESHNESS_BASE_DELAY * (2**(retries - 1)), MAX_FRESHNESS_DELAY].min
-            sleep(delay)
-          end
-        rescue => e
-          Braintrust::Log.warn("[BTQL] Query failed: #{e.message}")
-          []
+          execute_query(query: query, fmt: "jsonl")
         end
 
         private

data/lib/braintrust/eval/context.rb

@@ -9,11 +9,24 @@ module Braintrust
     class Context
       attr_reader :task, :scorers, :cases, :experiment_id, :experiment_name,
         :project_id, :project_name, :state, :tracer_provider,
-        :on_progress, :parent_span_attr, :generation
+        :on_progress, :parent_span_attr, :generation, :parameters
 
+      # @param task [Task] Normalized task wrapper
+      # @param scorers [Array<Scorer>] Normalized scorer wrappers
+      # @param cases [Cases] Normalized eval cases
+      # @param experiment_id [String, nil] Experiment ID for logging and trace linkage
+      # @param experiment_name [String, nil] Experiment name, included in span attributes
+      # @param project_id [String, nil] Project ID
+      # @param project_name [String, nil] Project name
+      # @param state [Braintrust::State, nil] Authenticated API state; nil for local-only evals
+      # @param tracer_provider [#tracer, nil] OpenTelemetry tracer provider
+      # @param on_progress [Proc, nil] Callback invoked after each case completes, receiving a progress Hash
+      # @param parent_span_attr [String, nil] Formatted parent span identifier ("type:id"), linking spans to a parent context
+      # @param generation [Integer, nil] Generation number from the parent span context, used to link spans in a trace hierarchy
+      # @param parameters [Hash, nil] Runtime parameters passed to task and scorers as a `parameters:` keyword argument
       def initialize(task:, scorers:, cases:, experiment_id: nil, experiment_name: nil,
         project_id: nil, project_name: nil, state: nil, tracer_provider: nil,
-        on_progress: nil, parent_span_attr: nil, generation: nil)
+        on_progress: nil, parent_span_attr: nil, generation: nil, parameters: nil)
         @task = task
         @scorers = scorers
         @cases = cases
@@ -26,40 +39,83 @@ module Braintrust
         @on_progress = on_progress
         @parent_span_attr = parent_span_attr
         @generation = generation
+        @parameters = parameters
       end
 
       # Build a Context from raw user inputs.
-      # Factory normalizes task, scorers, and cases into typed wrappers.
-      # Parent is resolved into parent_span_attr and generation.
+      # Delegates to Factory for normalization.
+      # @param task [Task, Proc, #call] Task to evaluate; wrapped into a {Task} if needed
+      # @param scorers [Array<Scorer, Proc, String, Scorer::ID, #call>] Scorers; each is normalized into a {Scorer}
+      # @param cases [Cases, Array, Enumerable] Eval cases; wrapped into {Cases} if needed
+      # @param experiment_id [String, nil] Experiment ID for logging
+      # @param experiment_name [String, nil] Experiment name, included in span attributes
+      # @param project_id [String, nil] Project ID
+      # @param project_name [String, nil] Project name; required when resolving scorer slugs
+      # @param state [Braintrust::State, nil] Authenticated API state; nil for local-only evals
+      # @param tracer_provider [#tracer, nil] OpenTelemetry tracer provider; defaults to global provider
+      # @param on_progress [Proc, nil] Callback invoked after each case completes, receiving a progress Hash
+      # @param parent [Hash, nil] Parent span info with keys :object_type, :object_id, and optionally :generation
+      # @param parameters [Hash, nil] Runtime parameters passed to task and scorers as a `parameters:` keyword argument
+      # @return [Context]
       def self.build(task:, scorers:, cases:, experiment_id: nil, experiment_name: nil,
         project_id: nil, project_name: nil, state: nil, tracer_provider: nil,
-        on_progress: nil, parent: nil)
-        factory = Factory.new(state: state, tracer_provider: tracer_provider, project_name: project_name)
-
-        Context.new(
-          task: factory.normalize_task(task),
-          scorers: factory.normalize_scorers(scorers),
-          cases: factory.normalize_cases(cases),
-          experiment_id: experiment_id,
-          experiment_name: experiment_name,
-          project_id: project_id,
-          project_name: project_name,
-          state: state,
-          tracer_provider: tracer_provider,
-          on_progress: on_progress,
-          parent_span_attr: factory.resolve_parent_span_attr(parent),
-          generation: parent&.dig(:generation)
+        on_progress: nil, parent: nil, parameters: nil)
+        Factory.new(
+          state: state, tracer_provider: tracer_provider,
+          project_id: project_id, project_name: project_name
+        ).build(
+          task: task, scorers: scorers, cases: cases,
+          experiment_id: experiment_id, experiment_name: experiment_name,
+          on_progress: on_progress, parent: parent, parameters: parameters
         )
       end
 
       # Encapsulates normalization of raw user inputs into typed wrappers.
       class Factory
-        def initialize(state: nil, tracer_provider: nil, project_name: nil)
+        # @param state [Braintrust::State, nil] Authenticated API state; passed through to scorer resolution
+        # @param tracer_provider [#tracer, nil] OpenTelemetry tracer provider; passed through to remote scorers
+        # @param project_id [String, nil] Project ID; passed through to the built Context
+        # @param project_name [String, nil] Project name; required when resolving scorer slugs
+        def initialize(state: nil, tracer_provider: nil, project_id: nil, project_name: nil)
           @state = state
           @tracer_provider = tracer_provider
+          @project_id = project_id
           @project_name = project_name
         end
 
+        # Normalize raw inputs and construct a {Context}.
+        # @param task [Task, Proc, #call] Raw task
+        # @param scorers [Array] Raw scorers
+        # @param cases [Cases, Array, Enumerable] Raw eval cases
+        # @param experiment_id [String, nil]
+        # @param experiment_name [String, nil]
+        # @param on_progress [Proc, nil]
+        # @param parent [Hash, nil] Parent span info with keys :object_type, :object_id, and optionally :generation
+        # @return [Context]
+        def build(task:, scorers:, cases:, experiment_id: nil, experiment_name: nil,
+          on_progress: nil, parent: nil, parameters: nil)
+          Context.new(
+            task: normalize_task(task),
+            scorers: normalize_scorers(scorers),
+            cases: normalize_cases(cases),
+            experiment_id: experiment_id,
+            experiment_name: experiment_name,
+            project_id: @project_id,
+            project_name: @project_name,
+            state: @state,
+            tracer_provider: @tracer_provider || OpenTelemetry.tracer_provider,
+            on_progress: on_progress,
+            parent_span_attr: resolve_parent_span_attr(parent),
+            generation: parent&.dig(:generation),
+            parameters: parameters
+          )
+        end
+
+        private
+
+        # @param raw [Cases, Array, Enumerable, #each]
+        # @return [Cases]
+        # @raise [ArgumentError] if raw is not enumerable
         def normalize_cases(raw)
           case raw
           when Cases
@@ -75,11 +131,15 @@ module Braintrust
           end
         end
 
+        # @param parent [Hash, nil]
+        # @return [String, nil] Formatted as "type:id", e.g. "experiment_id:abc-123"
         def resolve_parent_span_attr(parent)
           return nil unless parent
           "#{parent[:object_type]}:#{parent[:object_id]}"
         end
 
+        # @param raw [Task, Proc, #call]
+        # @return [Task]
         def normalize_task(raw)
           case raw
           when Task
@@ -95,6 +155,9 @@ module Braintrust
           end
         end
 
+        # @param raw [Array<Scorer, Proc, String, Scorer::ID, #call>]
+        # @return [Array<Scorer>]
+        # @raise [ArgumentError] if a String slug is given without a project name
         def normalize_scorers(raw)
           raw.map do |scorer|
             case scorer

data/lib/braintrust/eval/evaluator.rb

@@ -27,6 +27,18 @@ module Braintrust
     #       Braintrust::Scorer.new("exact_match") { |expected:, output:| output == expected ? 1.0 : 0.0 }
     #     ]
     #   )
+    #
+    # @example Remote eval with parameters (for Playground UI)
+    #   Braintrust::Eval::Evaluator.new(
+    #     task: ->(input:, parameters:) {
+    #       model = parameters["model"] || "gpt-4"
+    #       # Use model to generate response...
+    #     },
+    #     scorers: [Braintrust::Scorer.new("exact") { |expected:, output:| output == expected ? 1.0 : 0.0 }],
+    #     parameters: {
+    #       "model" => {type: "string", default: "gpt-4", description: "Model to use"}
+    #     }
+    #   )
     class Evaluator
       attr_accessor :task, :scorers, :parameters
 
@@ -64,13 +76,15 @@ module Braintrust
       def run(cases, on_progress: nil, quiet: false,
         project: nil, experiment: nil, project_id: nil,
         dataset: nil, scorers: nil, parent: nil,
-        state: nil, update: false, tracer_provider: nil)
+        state: nil, update: false, tracer_provider: nil,
+        parameters: nil)
         all_scorers = scorers ? self.scorers + scorers : self.scorers
         Braintrust::Eval.run(
           task: task, scorers: all_scorers, cases: cases, dataset: dataset,
           project: project, experiment: experiment, project_id: project_id,
           parent: parent, on_progress: on_progress, quiet: quiet,
-          state: state, update: update, tracer_provider: tracer_provider
+          state: state, update: update, tracer_provider: tracer_provider,
+          parameters: parameters
         )
       end
     end

data/lib/braintrust/eval/runner.rb

@@ -6,6 +6,7 @@ require_relative "summary"
 require_relative "trace"
 require_relative "../internal/thread_pool"
 require_relative "../api/internal/btql"
+require_relative "../internal/retry"
 
 require "opentelemetry/sdk"
 require "json"
@@ -24,8 +25,7 @@ module Braintrust
       # @param eval_context [Context] Normalized eval context
       def initialize(eval_context)
         @eval_context = eval_context
-        tracer_provider = eval_context.tracer_provider || OpenTelemetry.tracer_provider
-        @tracer = tracer_provider.tracer("braintrust-eval")
+        @tracer = eval_context.tracer_provider.tracer("braintrust-eval")
 
         # Mutex for thread-safe score collection
         @score_mutex = Mutex.new
@@ -79,50 +79,50 @@ module Braintrust
 
       # Run a single test case with OpenTelemetry tracing
       # Creates eval span (parent) with task and score as children
-      # @param case_context [CaseContext] The per-case accumulator
+      # @param kase [CaseContext] The per-case accumulator
       # @param errors [Queue] Thread-safe error collection queue
-      def run_eval_case(case_context, errors)
+      def run_eval_case(kase, errors)
         # Each eval case starts its own trace — detach from any ambient span context
         eval_span = tracer.start_root_span("eval")
         OpenTelemetry::Trace.with_span(eval_span) do
           # Set attributes known before task execution
           eval_span.set_attribute("braintrust.parent", eval_context.parent_span_attr) if eval_context.parent_span_attr
           set_json_attr(eval_span, "braintrust.span_attributes", build_span_attributes("eval"))
-          set_json_attr(eval_span, "braintrust.input_json", {input: case_context.input})
-          set_json_attr(eval_span, "braintrust.expected", case_context.expected) if case_context.expected
-          set_json_attr(eval_span, "braintrust.metadata", case_context.metadata) if case_context.metadata
-          eval_span.set_attribute("braintrust.tags", case_context.tags) if case_context.tags
-          eval_span.set_attribute("braintrust.origin", case_context.origin) if case_context.origin
+          set_json_attr(eval_span, "braintrust.input_json", {input: kase.input})
+          set_json_attr(eval_span, "braintrust.expected", kase.expected) if kase.expected
+          set_json_attr(eval_span, "braintrust.metadata", kase.metadata) if kase.metadata
+          eval_span.set_attribute("braintrust.tags", kase.tags) if kase.tags
+          eval_span.set_attribute("braintrust.origin", kase.origin) if kase.origin
 
           # Run task
           begin
-            case_context.output = run_task(case_context)
+            kase.output = run_task(kase)
           rescue => e
             # Error already recorded on task span, set eval span status
             eval_span.status = OpenTelemetry::Trace::Status.error(e.message)
             set_json_attr(eval_span, "braintrust.output_json", {output: nil})
-            errors << "Task failed for input '#{case_context.input}': #{e.message}"
-            report_progress(eval_span, case_context, error: e.message)
+            errors << "Task failed for input '#{kase.input}': #{e.message}"
+            report_progress(eval_span, kase, error: e.message)
             next
           end
 
           # Flush spans so they're queryable via BTQL, then build trace
-          eval_context.tracer_provider&.force_flush
-          case_context.trace = build_trace(eval_span)
+          eval_context.tracer_provider.force_flush if eval_context.tracer_provider.respond_to?(:force_flush)
+          kase.trace = build_trace(eval_span)
 
           # Run scorers
           begin
-            run_scorers(case_context)
+            run_scorers(kase)
           rescue => e
             # Error already recorded on score span, set eval span status
             eval_span.status = OpenTelemetry::Trace::Status.error(e.message)
-            errors << "Scorers failed for input '#{case_context.input}': #{e.message}"
+            errors << "Scorers failed for input '#{kase.input}': #{e.message}"
           end
 
           # Set output after task completes
-          set_json_attr(eval_span, "braintrust.output_json", {output: case_context.output})
+          set_json_attr(eval_span, "braintrust.output_json", {output: kase.output})
 
-          report_progress(eval_span, case_context, data: case_context.output)
+          report_progress(eval_span, kase, data: kase.output)
         end
       ensure
         eval_span&.finish
@@ -130,17 +130,18 @@ module Braintrust
 
       # Run task with OpenTelemetry tracing
       # Creates task span with input and output
-      # @param case_context [CaseContext] The per-case context
+      # @param kase [CaseContext] The per-case context
       # @return [Object] Task output
-      def run_task(case_context)
+      def run_task(kase)
         tracer.in_span("task") do |task_span|
           task_span.set_attribute("braintrust.parent", eval_context.parent_span_attr) if eval_context.parent_span_attr
           set_json_attr(task_span, "braintrust.span_attributes", build_span_attributes("task"))
-          set_json_attr(task_span, "braintrust.input_json", case_context.input)
+          set_json_attr(task_span, "braintrust.input_json", kase.input)
 
           begin
             output = eval_context.task.call(
-              input: case_context.input
+              input: kase.input,
+              parameters: eval_context.parameters || {}
             )
             set_json_attr(task_span, "braintrust.output_json", output)
             output
@@ -155,20 +156,22 @@ module Braintrust
 
       # Run scorers with OpenTelemetry tracing.
       # Creates one span per scorer, each a direct child of the current (eval) span.
-      # @param case_context [CaseContext] The per-case context (output must be populated)
-      def run_scorers(case_context)
+      # @param kase [CaseContext] The per-case context (output must be populated)
+      def run_scorers(kase)
         scorer_kwargs = {
-          input: case_context.input,
-          expected: case_context.expected,
-          output: case_context.output,
-          metadata: case_context.metadata || {},
-          trace: case_context.trace
+          input: kase.input,
+          expected: kase.expected,
+          output: kase.output,
+          metadata: kase.metadata || {},
+          trace: kase.trace,
+          parameters: eval_context.parameters || {}
         }
         scorer_input = {
-          input: case_context.input,
-          expected: case_context.expected,
-          output: case_context.output,
-          metadata: case_context.metadata || {}
+          input: kase.input,
+          expected: kase.expected,
+          output: kase.output,
+          metadata: kase.metadata || {},
+          parameters: eval_context.parameters || {}
         }
 
         scorer_error = nil
@@ -224,9 +227,23 @@ module Braintrust
         object_id = eval_context.experiment_id
         btql = API::Internal::BTQL.new(eval_context.state)
 
-        Eval::Trace.new(
-          spans: -> { btql.trace_spans(object_type: object_type, object_id: object_id, root_span_id: root_span_id) }
-        )
+        Eval::Trace.new(spans: -> { fetch_trace_spans(btql, object_type, object_id, root_span_id) })
+      end
+
+      # Fetch trace spans with retry to handle freshness and ingestion lag.
+      # @return [Array<Hash>] Parsed span data
+      def fetch_trace_spans(btql, object_type, object_id, root_span_id)
+        rows, _freshness = Internal::Retry.with_backoff(
+          max_retries: 7, base_delay: 1.0, max_delay: 8.0,
+          until: ->(result) {
+            r, f = result
+            f == "complete" && !r.empty?
+          }
+        ) { btql.trace_spans(object_type: object_type, object_id: object_id, root_span_id: root_span_id) }
+        rows || []
+      rescue => e
+        Braintrust::Log.warn("[BTQL] Query failed: #{e.message}")
+        []
       end
 
       # Build a CaseContext from a Case struct
@@ -241,11 +258,11 @@ module Braintrust
 
       # Report progress for a case via on_progress callback.
       # Rescues errors in the callback so a broken handler never crashes the eval.
-      def report_progress(eval_span, case_context, **fields)
+      def report_progress(eval_span, kase, **fields)
         return unless eval_context.on_progress
         progress = {"id" => eval_span.context.hex_span_id}.merge(fields.transform_keys(&:to_s))
-        if case_context.origin
-          progress["origin"] = case_context.origin.is_a?(String) ? JSON.parse(case_context.origin) : case_context.origin
+        if kase.origin
+          progress["origin"] = kase.origin.is_a?(String) ? JSON.parse(kase.origin) : kase.origin
         end
         eval_context.on_progress.call(progress)
       rescue => e

data/lib/braintrust/eval.rb

@@ -105,6 +105,21 @@ module Braintrust
   #     scorers: [->(expected:, output:) { output == expected ? 1.0 : 0.0 }]
   #   )
   #
+  # @example Using parameters for configurable tasks
+  #   # Tasks and scorers that declare `parameters:` receive it automatically.
+  #   # Those that don't are unaffected — KeywordFilter strips unknown kwargs.
+  #   Braintrust::Eval.run(
+  #     project: "my-project",
+  #     experiment: "with-params",
+  #     cases: [{input: "hello", expected: "HELLO!"}],
+  #     task: ->(input:, parameters:) {
+  #       suffix = parameters["suffix"] || ""
+  #       input.upcase + suffix
+  #     },
+  #     scorers: [->(expected:, output:) { output == expected ? 1.0 : 0.0 }],
+  #     parameters: {"suffix" => "!"}
+  #   )
+  #
   # @example Using metadata and tags
   #   Braintrust::Eval.run(
   #     project: "my-project",
@@ -158,11 +173,15 @@ module Braintrust
       # @param quiet [Boolean] If true, suppress result output (default: false)
       # @param state [State, nil] Braintrust state (defaults to global state)
       # @param tracer_provider [TracerProvider, nil] OpenTelemetry tracer provider (defaults to global)
+      # @param project_id [String, nil] Project UUID (skips project creation when provided)
+      # @param parent [Hash, nil] Parent span context ({object_type:, object_id:, generation:})
+      # @param parameters [Hash, nil] Runtime parameters passed to task and scorers as a `parameters:` keyword argument
       # @return [Result]
       def run(task:, scorers:, project: nil, experiment: nil,
         cases: nil, dataset: nil, on_progress: nil,
         parallelism: 1, tags: nil, metadata: nil, update: false, quiet: false,
-        state: nil, tracer_provider: nil, project_id: nil, parent: nil)
+        state: nil, tracer_provider: nil, project_id: nil, parent: nil,
+        parameters: nil)
         # Validate required parameters
         validate_params!(task: task, scorers: scorers, cases: cases, dataset: dataset)
 
@@ -205,7 +224,8 @@ module Braintrust
           state: state,
           tracer_provider: tracer_provider,
           on_progress: on_progress,
-          parent: parent
+          parent: parent,
+          parameters: parameters
         )
         result = Runner.new(context).run(parallelism: parallelism)
 

data/lib/braintrust/internal/retry.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Internal
+    module Retry
+      MAX_RETRIES = 7
+      BASE_DELAY = 1.0
+      MAX_DELAY = 8.0
+
+      # Retry a block with exponential backoff.
+      #
+      # The block is the task to attempt. Its return value is captured each attempt.
+      #
+      # @param max_retries [Integer] Maximum number of retries after the first attempt
+      # @param base_delay [Float] Initial delay in seconds (doubles each retry)
+      # @param max_delay [Float] Cap on delay between retries
+      # @param until [Proc, nil] Optional condition — receives block result, truthy stops retrying.
+      #   When omitted, the block result's own truthiness decides.
+      # @return The last block result (whether retries were exhausted or condition was met)
+      #
+      # @example Simple: retry until truthy
+      #   conn = Retry.with_backoff(max_retries: 5) { try_connect }
+      #
+      # @example With condition: retry until non-empty
+      #   data = Retry.with_backoff(until: ->(r) { r.any? }) { api.fetch }
+      #
+      def self.with_backoff(max_retries: MAX_RETRIES, base_delay: BASE_DELAY, max_delay: MAX_DELAY, until: nil, &task)
+        check = binding.local_variable_get(:until)
+        result = task.call
+        retries = 0
+        while retries < max_retries && !(check ? check.call(result) : result)
+          retries += 1
+          delay = [base_delay * (2**(retries - 1)), max_delay].min
+          sleep(delay)
+          result = task.call
+        end
+        result
+      end
+    end
+  end
+end

data/lib/braintrust/prompt.rb

@@ -11,23 +11,28 @@ module Braintrust
   #   params = prompt.build(text: "Article to summarize...")
   #   client.messages.create(**params)
   class Prompt
-    attr_reader :id, :name, :slug, :project_id
+    attr_reader :id, :name, :slug, :project_id, :version
 
     # Load a prompt from Braintrust
     #
-    # @param project [String] Project name
+    # @param project [String, nil] Project name (provide either project or project_id)
+    # @param project_id [String, nil] Project ID (UUID, provide either project or project_id)
     # @param slug [String] Prompt slug
     # @param version [String, nil] Specific version (default: latest)
     # @param defaults [Hash] Default variable values for build()
     # @param api [API, nil] Braintrust API client (default: creates one using global state)
     # @return [Prompt]
-    def self.load(project:, slug:, version: nil, defaults: {}, api: nil)
+    def self.load(slug:, project: nil, project_id: nil, version: nil, defaults: {}, api: nil)
+      raise ArgumentError, "Either project or project_id is required" unless project || project_id
+
       api ||= API.new
 
       # Find the function by project + slug
-      result = api.functions.list(project_name: project, slug: slug)
+      result = api.functions.list(project_name: project, project_id: project_id, slug: slug)
       function = result.dig("objects")&.first
-      raise Error, "Prompt '#{slug}' not found in project '#{project}'" unless function
+
+      identifier = project ? "project '#{project}'" : "project_id '#{project_id}'"
+      raise Error, "Prompt '#{slug}' not found in #{identifier}" unless function
 
       # Fetch full function data including prompt_data
       full_data = api.functions.get(id: function["id"], version: version)
@@ -47,6 +52,7 @@ module Braintrust
       @name = data["name"]
       @slug = data["slug"]
       @project_id = data["project_id"]
+      @version = data["_xact_id"]
     end
 
     # Get the raw prompt definition

data/lib/braintrust/server/services/eval_service.rb

@@ -40,7 +40,8 @@ module Braintrust
             experiment_name: body["experiment_name"],
             remote_scorer_ids: resolve_remote_scorers(body["scores"]),
             parent: resolve_parent(body["parent"]),
-            project_id: body["project_id"]
+            project_id: body["project_id"],
+            parameters: resolve_parameters(body["parameters"], evaluator)
           }
         end
 
@@ -57,6 +58,7 @@ module Braintrust
           remote_scorer_ids = validated[:remote_scorer_ids]
           parent = validated[:parent]
           project_id = validated[:project_id]
+          parameters = validated[:parameters]
 
           state = build_state(auth)
 
@@ -89,6 +91,7 @@ module Braintrust
           }
           run_opts[:parent] = parent if parent
           run_opts[:scorers] = remote_scorer_ids if remote_scorer_ids
+          run_opts[:parameters] = parameters if parameters && !parameters.empty?
           run_opts[:dataset] = dataset if dataset
 
           if state
@@ -161,6 +164,15 @@ module Braintrust
           @evaluators
         end
 
+        # Merge request parameters with evaluator's parameter defaults.
+        # Request values override defaults. Returns a string-keyed Hash.
+        def resolve_parameters(raw_params, evaluator)
+          defaults = (evaluator.parameters || {}).to_h { |name, spec|
+            [name.to_s, spec.is_a?(Hash) ? (spec[:default] || spec["default"]) : nil]
+          }.compact
+          defaults.merge(raw_params || {})
+        end
+
         # Resolve data source from the data field.
         # Returns [cases, dataset] where exactly one is non-nil.
         def resolve_data_source(data)

data/lib/braintrust/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Braintrust
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: braintrust
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Braintrust
@@ -9,6 +9,20 @@ bindir: exe
 cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: opentelemetry-sdk
   requirement: !ruby/object:Gem::Requirement
@@ -51,132 +65,6 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.3.1
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.0'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-- !ruby/object:Gem::Dependency
-  name: standard
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.0'
-- !ruby/object:Gem::Dependency
-  name: simplecov
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.22'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.22'
-- !ruby/object:Gem::Dependency
-  name: vcr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '6.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '6.0'
-- !ruby/object:Gem::Dependency
-  name: webmock
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-- !ruby/object:Gem::Dependency
-  name: appraisal
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.5'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.5'
-- !ruby/object:Gem::Dependency
-  name: yard
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.9'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.9'
-- !ruby/object:Gem::Dependency
-  name: kramdown
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.0'
 description: 'Braintrust Ruby SDK for evals, tracing and more. '
 email:
 - info@braintrust.dev
@@ -258,6 +146,7 @@ files:
 - lib/braintrust/internal/env.rb
 - lib/braintrust/internal/http.rb
 - lib/braintrust/internal/origin.rb
+- lib/braintrust/internal/retry.rb
 - lib/braintrust/internal/template.rb
 - lib/braintrust/internal/thread_pool.rb
 - lib/braintrust/internal/time.rb