braintrust 0.2.1 → 0.3.1

data/lib/braintrust/eval/context.rb

@@ -9,11 +9,24 @@ module Braintrust
     class Context
       attr_reader :task, :scorers, :cases, :experiment_id, :experiment_name,
         :project_id, :project_name, :state, :tracer_provider,
-        :on_progress, :parent_span_attr, :generation
+        :on_progress, :parent_span_attr, :generation, :parameters
 
+      # @param task [Task] Normalized task wrapper
+      # @param scorers [Array<Scorer>] Normalized scorer wrappers
+      # @param cases [Cases] Normalized eval cases
+      # @param experiment_id [String, nil] Experiment ID for logging and trace linkage
+      # @param experiment_name [String, nil] Experiment name, included in span attributes
+      # @param project_id [String, nil] Project ID
+      # @param project_name [String, nil] Project name
+      # @param state [Braintrust::State, nil] Authenticated API state; nil for local-only evals
+      # @param tracer_provider [#tracer, nil] OpenTelemetry tracer provider
+      # @param on_progress [Proc, nil] Callback invoked after each case completes, receiving a progress Hash
+      # @param parent_span_attr [String, nil] Formatted parent span identifier ("type:id"), linking spans to a parent context
+      # @param generation [Integer, nil] Generation number from the parent span context, used to link spans in a trace hierarchy
+      # @param parameters [Hash, nil] Runtime parameters passed to task and scorers as a `parameters:` keyword argument
       def initialize(task:, scorers:, cases:, experiment_id: nil, experiment_name: nil,
         project_id: nil, project_name: nil, state: nil, tracer_provider: nil,
-        on_progress: nil, parent_span_attr: nil, generation: nil)
+        on_progress: nil, parent_span_attr: nil, generation: nil, parameters: nil)
         @task = task
         @scorers = scorers
         @cases = cases
@@ -26,40 +39,83 @@ module Braintrust
         @on_progress = on_progress
         @parent_span_attr = parent_span_attr
         @generation = generation
+        @parameters = parameters
       end
 
       # Build a Context from raw user inputs.
-      # Factory normalizes task, scorers, and cases into typed wrappers.
-      # Parent is resolved into parent_span_attr and generation.
+      # Delegates to Factory for normalization.
+      # @param task [Task, Proc, #call] Task to evaluate; wrapped into a {Task} if needed
+      # @param scorers [Array<Scorer, Proc, String, Scorer::ID, #call>] Scorers; each is normalized into a {Scorer}
+      # @param cases [Cases, Array, Enumerable] Eval cases; wrapped into {Cases} if needed
+      # @param experiment_id [String, nil] Experiment ID for logging
+      # @param experiment_name [String, nil] Experiment name, included in span attributes
+      # @param project_id [String, nil] Project ID
+      # @param project_name [String, nil] Project name; required when resolving scorer slugs
+      # @param state [Braintrust::State, nil] Authenticated API state; nil for local-only evals
+      # @param tracer_provider [#tracer, nil] OpenTelemetry tracer provider; defaults to global provider
+      # @param on_progress [Proc, nil] Callback invoked after each case completes, receiving a progress Hash
+      # @param parent [Hash, nil] Parent span info with keys :object_type, :object_id, and optionally :generation
+      # @param parameters [Hash, nil] Runtime parameters passed to task and scorers as a `parameters:` keyword argument
+      # @return [Context]
       def self.build(task:, scorers:, cases:, experiment_id: nil, experiment_name: nil,
         project_id: nil, project_name: nil, state: nil, tracer_provider: nil,
-        on_progress: nil, parent: nil)
-        factory = Factory.new(state: state, tracer_provider: tracer_provider, project_name: project_name)
-
-        Context.new(
-          task: factory.normalize_task(task),
-          scorers: factory.normalize_scorers(scorers),
-          cases: factory.normalize_cases(cases),
-          experiment_id: experiment_id,
-          experiment_name: experiment_name,
-          project_id: project_id,
-          project_name: project_name,
-          state: state,
-          tracer_provider: tracer_provider,
-          on_progress: on_progress,
-          parent_span_attr: factory.resolve_parent_span_attr(parent),
-          generation: parent&.dig(:generation)
+        on_progress: nil, parent: nil, parameters: nil)
+        Factory.new(
+          state: state, tracer_provider: tracer_provider,
+          project_id: project_id, project_name: project_name
+        ).build(
+          task: task, scorers: scorers, cases: cases,
+          experiment_id: experiment_id, experiment_name: experiment_name,
+          on_progress: on_progress, parent: parent, parameters: parameters
         )
       end
 
       # Encapsulates normalization of raw user inputs into typed wrappers.
       class Factory
-        def initialize(state: nil, tracer_provider: nil, project_name: nil)
+        # @param state [Braintrust::State, nil] Authenticated API state; passed through to scorer resolution
+        # @param tracer_provider [#tracer, nil] OpenTelemetry tracer provider; passed through to remote scorers
+        # @param project_id [String, nil] Project ID; passed through to the built Context
+        # @param project_name [String, nil] Project name; required when resolving scorer slugs
+        def initialize(state: nil, tracer_provider: nil, project_id: nil, project_name: nil)
           @state = state
           @tracer_provider = tracer_provider
+          @project_id = project_id
           @project_name = project_name
         end
 
+        # Normalize raw inputs and construct a {Context}.
+        # @param task [Task, Proc, #call] Raw task
+        # @param scorers [Array] Raw scorers
+        # @param cases [Cases, Array, Enumerable] Raw eval cases
+        # @param experiment_id [String, nil]
+        # @param experiment_name [String, nil]
+        # @param on_progress [Proc, nil]
+        # @param parent [Hash, nil] Parent span info with keys :object_type, :object_id, and optionally :generation
+        # @return [Context]
+        def build(task:, scorers:, cases:, experiment_id: nil, experiment_name: nil,
+          on_progress: nil, parent: nil, parameters: nil)
+          Context.new(
+            task: normalize_task(task),
+            scorers: normalize_scorers(scorers),
+            cases: normalize_cases(cases),
+            experiment_id: experiment_id,
+            experiment_name: experiment_name,
+            project_id: @project_id,
+            project_name: @project_name,
+            state: @state,
+            tracer_provider: @tracer_provider || OpenTelemetry.tracer_provider,
+            on_progress: on_progress,
+            parent_span_attr: resolve_parent_span_attr(parent),
+            generation: parent&.dig(:generation),
+            parameters: parameters
+          )
+        end
+
+        private
+
+        # @param raw [Cases, Array, Enumerable, #each]
+        # @return [Cases]
+        # @raise [ArgumentError] if raw is not enumerable
         def normalize_cases(raw)
           case raw
           when Cases
@@ -75,11 +131,15 @@ module Braintrust
           end
         end
 
+        # @param parent [Hash, nil]
+        # @return [String, nil] Formatted as "type:id", e.g. "experiment_id:abc-123"
         def resolve_parent_span_attr(parent)
           return nil unless parent
           "#{parent[:object_type]}:#{parent[:object_id]}"
         end
 
+        # @param raw [Task, Proc, #call]
+        # @return [Task]
         def normalize_task(raw)
           case raw
           when Task
@@ -95,6 +155,9 @@ module Braintrust
           end
         end
 
+        # @param raw [Array<Scorer, Proc, String, Scorer::ID, #call>]
+        # @return [Array<Scorer>]
+        # @raise [ArgumentError] if a String slug is given without a project name
         def normalize_scorers(raw)
           raw.map do |scorer|
             case scorer

data/lib/braintrust/eval/evaluator.rb

@@ -27,6 +27,18 @@ module Braintrust
     #       Braintrust::Scorer.new("exact_match") { |expected:, output:| output == expected ? 1.0 : 0.0 }
     #     ]
     #   )
+    #
+    # @example Remote eval with parameters (for Playground UI)
+    #   Braintrust::Eval::Evaluator.new(
+    #     task: ->(input:, parameters:) {
+    #       model = parameters["model"] || "gpt-4"
+    #       # Use model to generate response...
+    #     },
+    #     scorers: [Braintrust::Scorer.new("exact") { |expected:, output:| output == expected ? 1.0 : 0.0 }],
+    #     parameters: {
+    #       "model" => {type: "string", default: "gpt-4", description: "Model to use"}
+    #     }
+    #   )
     class Evaluator
       attr_accessor :task, :scorers, :parameters
 
@@ -64,13 +76,15 @@ module Braintrust
       def run(cases, on_progress: nil, quiet: false,
         project: nil, experiment: nil, project_id: nil,
         dataset: nil, scorers: nil, parent: nil,
-        state: nil, update: false, tracer_provider: nil)
+        state: nil, update: false, tracer_provider: nil,
+        parameters: nil)
         all_scorers = scorers ? self.scorers + scorers : self.scorers
         Braintrust::Eval.run(
           task: task, scorers: all_scorers, cases: cases, dataset: dataset,
           project: project, experiment: experiment, project_id: project_id,
           parent: parent, on_progress: on_progress, quiet: quiet,
-          state: state, update: update, tracer_provider: tracer_provider
+          state: state, update: update, tracer_provider: tracer_provider,
+          parameters: parameters
         )
       end
     end

data/lib/braintrust/eval/runner.rb

@@ -6,6 +6,7 @@ require_relative "summary"
 require_relative "trace"
 require_relative "../internal/thread_pool"
 require_relative "../api/internal/btql"
+require_relative "../internal/retry"
 
 require "opentelemetry/sdk"
 require "json"
@@ -24,8 +25,7 @@ module Braintrust
       # @param eval_context [Context] Normalized eval context
       def initialize(eval_context)
         @eval_context = eval_context
-        tracer_provider = eval_context.tracer_provider || OpenTelemetry.tracer_provider
-        @tracer = tracer_provider.tracer("braintrust-eval")
+        @tracer = eval_context.tracer_provider.tracer("braintrust-eval")
 
         # Mutex for thread-safe score collection
         @score_mutex = Mutex.new
@@ -79,66 +79,69 @@ module Braintrust
 
       # Run a single test case with OpenTelemetry tracing
       # Creates eval span (parent) with task and score as children
-      # @param case_context [CaseContext] The per-case accumulator
+      # @param kase [CaseContext] The per-case accumulator
       # @param errors [Queue] Thread-safe error collection queue
-      def run_eval_case(case_context, errors)
-        tracer.in_span("eval") do |eval_span|
+      def run_eval_case(kase, errors)
+        # Each eval case starts its own trace — detach from any ambient span context
+        eval_span = tracer.start_root_span("eval")
+        OpenTelemetry::Trace.with_span(eval_span) do
+          # Set attributes known before task execution
           eval_span.set_attribute("braintrust.parent", eval_context.parent_span_attr) if eval_context.parent_span_attr
-
-          # Set tags early so they're present even if task fails
-          eval_span.set_attribute("braintrust.tags", case_context.tags) if case_context.tags
+          set_json_attr(eval_span, "braintrust.span_attributes", build_span_attributes("eval"))
+          set_json_attr(eval_span, "braintrust.input_json", {input: kase.input})
+          set_json_attr(eval_span, "braintrust.expected", kase.expected) if kase.expected
+          set_json_attr(eval_span, "braintrust.metadata", kase.metadata) if kase.metadata
+          eval_span.set_attribute("braintrust.tags", kase.tags) if kase.tags
+          eval_span.set_attribute("braintrust.origin", kase.origin) if kase.origin
 
           # Run task
           begin
-            case_context.output = run_task(case_context)
+            kase.output = run_task(kase)
           rescue => e
             # Error already recorded on task span, set eval span status
             eval_span.status = OpenTelemetry::Trace::Status.error(e.message)
-            errors << "Task failed for input '#{case_context.input}': #{e.message}"
-            report_progress(eval_span, case_context, error: e.message)
+            set_json_attr(eval_span, "braintrust.output_json", {output: nil})
+            errors << "Task failed for input '#{kase.input}': #{e.message}"
+            report_progress(eval_span, kase, error: e.message)
             next
           end
 
           # Flush spans so they're queryable via BTQL, then build trace
-          eval_context.tracer_provider&.force_flush
-          case_context.trace = build_trace(eval_span)
+          eval_context.tracer_provider.force_flush if eval_context.tracer_provider.respond_to?(:force_flush)
+          kase.trace = build_trace(eval_span)
 
           # Run scorers
-          case_scores = nil
           begin
-            case_scores = run_scorers(case_context)
+            run_scorers(kase)
           rescue => e
             # Error already recorded on score span, set eval span status
             eval_span.status = OpenTelemetry::Trace::Status.error(e.message)
-            errors << "Scorers failed for input '#{case_context.input}': #{e.message}"
+            errors << "Scorers failed for input '#{kase.input}': #{e.message}"
           end
 
-          # Set eval span attributes (after task and scorers complete)
-          set_json_attr(eval_span, "braintrust.span_attributes", build_span_attributes("eval"))
-          set_json_attr(eval_span, "braintrust.input_json", case_context.input)
-          set_json_attr(eval_span, "braintrust.output_json", case_context.output)
-          set_json_attr(eval_span, "braintrust.expected", case_context.expected) if case_context.expected
-
-          # Set origin for cases from remote sources (already JSON-serialized)
-          eval_span.set_attribute("braintrust.origin", case_context.origin) if case_context.origin
+          # Set output after task completes
+          set_json_attr(eval_span, "braintrust.output_json", {output: kase.output})
 
-          report_progress(eval_span, case_context, data: case_context.output, scores: case_scores || {})
+          report_progress(eval_span, kase, data: kase.output)
         end
+      ensure
+        eval_span&.finish
       end
 
       # Run task with OpenTelemetry tracing
       # Creates task span with input and output
-      # @param case_context [CaseContext] The per-case context
+      # @param kase [CaseContext] The per-case context
       # @return [Object] Task output
-      def run_task(case_context)
+      def run_task(kase)
         tracer.in_span("task") do |task_span|
           task_span.set_attribute("braintrust.parent", eval_context.parent_span_attr) if eval_context.parent_span_attr
           set_json_attr(task_span, "braintrust.span_attributes", build_span_attributes("task"))
-          set_json_attr(task_span, "braintrust.input_json", case_context.input)
+          set_json_attr(task_span, "braintrust.input_json", kase.input)
 
           begin
             output = eval_context.task.call(
-              input: case_context.input
+              input: kase.input,
+              parameters: eval_context.parameters || {}
             )
             set_json_attr(task_span, "braintrust.output_json", output)
             output
@@ -151,43 +154,64 @@ module Braintrust
         end
       end
 
-      # Run scorers with OpenTelemetry tracing
-      # Creates single score span for all scorers
-      # @param case_context [CaseContext] The per-case context (output must be populated)
-      # @return [Hash] Scores hash { scorer_name => score_value }
-      def run_scorers(case_context)
-        tracer.in_span("score") do |score_span|
+      # Run scorers with OpenTelemetry tracing.
+      # Creates one span per scorer, each a direct child of the current (eval) span.
+      # @param kase [CaseContext] The per-case context (output must be populated)
+      def run_scorers(kase)
+        scorer_kwargs = {
+          input: kase.input,
+          expected: kase.expected,
+          output: kase.output,
+          metadata: kase.metadata || {},
+          trace: kase.trace,
+          parameters: eval_context.parameters || {}
+        }
+        scorer_input = {
+          input: kase.input,
+          expected: kase.expected,
+          output: kase.output,
+          metadata: kase.metadata || {},
+          parameters: eval_context.parameters || {}
+        }
+
+        scorer_error = nil
+        eval_context.scorers.each do |scorer|
+          collect_scores(run_scorer(scorer, scorer_kwargs, scorer_input))
+        rescue => e
+          scorer_error ||= e
+        end
+
+        raise scorer_error if scorer_error
+      end
+
+      # Run a single scorer inside its own span.
+      # @param scorer [Scorer] The scorer to run
+      # @param scorer_kwargs [Hash] Keyword arguments for the scorer
+      # @param scorer_input [Hash] Input to log on the span
+      # @return [Array<Hash>] Raw score results from the scorer
+      def run_scorer(scorer, scorer_kwargs, scorer_input)
+        tracer.in_span(scorer.name) do |score_span|
           score_span.set_attribute("braintrust.parent", eval_context.parent_span_attr) if eval_context.parent_span_attr
-          set_json_attr(score_span, "braintrust.span_attributes", build_span_attributes("score"))
-
-          scorer_kwargs = {
-            input: case_context.input,
-            expected: case_context.expected,
-            output: case_context.output,
-            metadata: case_context.metadata || {},
-            trace: case_context.trace
-          }
-          scores = {}
-          scorer_error = nil
-          eval_context.scorers.each do |scorer|
-            score_value = scorer.call(**scorer_kwargs)
-            scores[scorer.name] = score_value
-
-            # Collect raw score for summary (thread-safe)
-            collect_score(scorer.name, score_value)
-          rescue => e
-            # Record first error but continue processing other scorers
-            scorer_error ||= e
-            record_span_error(score_span, e, "ScorerError")
-          end
+          set_json_attr(score_span, "braintrust.span_attributes", build_scorer_span_attributes(scorer.name))
+          set_json_attr(score_span, "braintrust.input_json", scorer_input)
+
+          score_results = scorer.call(**scorer_kwargs)
 
-          # Always set scores attribute, even if some scorers failed
-          set_json_attr(score_span, "braintrust.scores", scores)
+          scorer_scores = {}
+          scorer_metadata = {}
+          score_results.each do |s|
+            scorer_scores[s[:name]] = s[:score]
+            scorer_metadata[s[:name]] = s[:metadata] if s[:metadata].is_a?(Hash)
+          end
 
-          # Raise after setting scores so we can see which scorers succeeded
-          raise scorer_error if scorer_error
+          set_json_attr(score_span, "braintrust.output_json", scorer_scores)
+          set_json_attr(score_span, "braintrust.scores", scorer_scores)
+          set_json_attr(score_span, "braintrust.metadata", scorer_metadata) unless scorer_metadata.empty?
 
-          scores
+          score_results
+        rescue => e
+          record_span_error(score_span, e, "ScorerError")
+          raise
         end
       end
 
@@ -203,9 +227,23 @@ module Braintrust
         object_id = eval_context.experiment_id
         btql = API::Internal::BTQL.new(eval_context.state)
 
-        Eval::Trace.new(
-          spans: -> { btql.trace_spans(object_type: object_type, object_id: object_id, root_span_id: root_span_id) }
-        )
+        Eval::Trace.new(spans: -> { fetch_trace_spans(btql, object_type, object_id, root_span_id) })
+      end
+
+      # Fetch trace spans with retry to handle freshness and ingestion lag.
+      # @return [Array<Hash>] Parsed span data
+      def fetch_trace_spans(btql, object_type, object_id, root_span_id)
+        rows, _freshness = Internal::Retry.with_backoff(
+          max_retries: 7, base_delay: 1.0, max_delay: 8.0,
+          until: ->(result) {
+            r, f = result
+            f == "complete" && !r.empty?
+          }
+        ) { btql.trace_spans(object_type: object_type, object_id: object_id, root_span_id: root_span_id) }
+        rows || []
+      rescue => e
+        Braintrust::Log.warn("[BTQL] Query failed: #{e.message}")
+        []
       end
 
       # Build a CaseContext from a Case struct
@@ -220,11 +258,11 @@ module Braintrust
 
       # Report progress for a case via on_progress callback.
       # Rescues errors in the callback so a broken handler never crashes the eval.
-      def report_progress(eval_span, case_context, **fields)
+      def report_progress(eval_span, kase, **fields)
         return unless eval_context.on_progress
         progress = {"id" => eval_span.context.hex_span_id}.merge(fields.transform_keys(&:to_s))
-        if case_context.origin
-          progress["origin"] = case_context.origin.is_a?(String) ? JSON.parse(case_context.origin) : case_context.origin
+        if kase.origin
+          progress["origin"] = kase.origin.is_a?(String) ? JSON.parse(kase.origin) : kase.origin
         end
         eval_context.on_progress.call(progress)
       rescue => e
@@ -255,6 +293,16 @@ module Braintrust
         attrs
       end
 
+      # Build span_attributes for a scorer span.
+      # Each scorer gets its own span with type "score", purpose "scorer", and the scorer's name.
+      # @param scorer_name [String] The scorer name
+      # @return [Hash]
+      def build_scorer_span_attributes(scorer_name)
+        attrs = {type: "score", name: scorer_name, purpose: "scorer"}
+        attrs[:generation] = eval_context.generation if eval_context.generation
+        attrs
+      end
+
       # Set a span attribute by JSON encoding the value
       # @param span [OpenTelemetry::Trace::Span] The span
       # @param key [String] The attribute key
@@ -263,14 +311,11 @@ module Braintrust
         span.set_attribute(key, JSON.dump(value))
       end
 
-      # Collect a single score value for summary calculation
-      # @param name [String] Scorer name
-      # @param value [Object] Score value (only Numeric values are collected)
-      def collect_score(name, value)
-        return unless value.is_a?(Numeric)
-
+      # Collect score results into the summary accumulator (thread-safe).
+      # @param score_results [Array<Hash>] Score results from a scorer
+      def collect_scores(score_results)
         @score_mutex.synchronize do
-          (@scores[name] ||= []) << value
+          score_results.each { |s| (@scores[s[:name]] ||= []) << s[:score] }
         end
       end
     end

data/lib/braintrust/eval.rb

@@ -105,6 +105,21 @@ module Braintrust
   #     scorers: [->(expected:, output:) { output == expected ? 1.0 : 0.0 }]
   #   )
   #
+  # @example Using parameters for configurable tasks
+  #   # Tasks and scorers that declare `parameters:` receive it automatically.
+  #   # Those that don't are unaffected — KeywordFilter strips unknown kwargs.
+  #   Braintrust::Eval.run(
+  #     project: "my-project",
+  #     experiment: "with-params",
+  #     cases: [{input: "hello", expected: "HELLO!"}],
+  #     task: ->(input:, parameters:) {
+  #       suffix = parameters["suffix"] || ""
+  #       input.upcase + suffix
+  #     },
+  #     scorers: [->(expected:, output:) { output == expected ? 1.0 : 0.0 }],
+  #     parameters: {"suffix" => "!"}
+  #   )
+  #
   # @example Using metadata and tags
   #   Braintrust::Eval.run(
   #     project: "my-project",
@@ -158,11 +173,15 @@ module Braintrust
       # @param quiet [Boolean] If true, suppress result output (default: false)
       # @param state [State, nil] Braintrust state (defaults to global state)
       # @param tracer_provider [TracerProvider, nil] OpenTelemetry tracer provider (defaults to global)
+      # @param project_id [String, nil] Project UUID (skips project creation when provided)
+      # @param parent [Hash, nil] Parent span context ({object_type:, object_id:, generation:})
+      # @param parameters [Hash, nil] Runtime parameters passed to task and scorers as a `parameters:` keyword argument
       # @return [Result]
       def run(task:, scorers:, project: nil, experiment: nil,
         cases: nil, dataset: nil, on_progress: nil,
         parallelism: 1, tags: nil, metadata: nil, update: false, quiet: false,
-        state: nil, tracer_provider: nil, project_id: nil, parent: nil)
+        state: nil, tracer_provider: nil, project_id: nil, parent: nil,
+        parameters: nil)
         # Validate required parameters
         validate_params!(task: task, scorers: scorers, cases: cases, dataset: dataset)
 
@@ -205,7 +224,8 @@ module Braintrust
           state: state,
           tracer_provider: tracer_provider,
           on_progress: on_progress,
-          parent: parent
+          parent: parent,
+          parameters: parameters
         )
         result = Runner.new(context).run(parallelism: parallelism)
 

data/lib/braintrust/internal/retry.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Internal
+    module Retry
+      MAX_RETRIES = 7
+      BASE_DELAY = 1.0
+      MAX_DELAY = 8.0
+
+      # Retry a block with exponential backoff.
+      #
+      # The block is the task to attempt. Its return value is captured each attempt.
+      #
+      # @param max_retries [Integer] Maximum number of retries after the first attempt
+      # @param base_delay [Float] Initial delay in seconds (doubles each retry)
+      # @param max_delay [Float] Cap on delay between retries
+      # @param until [Proc, nil] Optional condition — receives block result, truthy stops retrying.
+      #   When omitted, the block result's own truthiness decides.
+      # @return The last block result (whether retries were exhausted or condition was met)
+      #
+      # @example Simple: retry until truthy
+      #   conn = Retry.with_backoff(max_retries: 5) { try_connect }
+      #
+      # @example With condition: retry until non-empty
+      #   data = Retry.with_backoff(until: ->(r) { r.any? }) { api.fetch }
+      #
+      def self.with_backoff(max_retries: MAX_RETRIES, base_delay: BASE_DELAY, max_delay: MAX_DELAY, until: nil, &task)
+        check = binding.local_variable_get(:until)
+        result = task.call
+        retries = 0
+        while retries < max_retries && !(check ? check.call(result) : result)
+          retries += 1
+          delay = [base_delay * (2**(retries - 1)), max_delay].min
+          sleep(delay)
+          result = task.call
+        end
+        result
+      end
+    end
+  end
+end

data/lib/braintrust/prompt.rb

@@ -11,23 +11,28 @@ module Braintrust
   #   params = prompt.build(text: "Article to summarize...")
   #   client.messages.create(**params)
   class Prompt
-    attr_reader :id, :name, :slug, :project_id
+    attr_reader :id, :name, :slug, :project_id, :version
 
     # Load a prompt from Braintrust
     #
-    # @param project [String] Project name
+    # @param project [String, nil] Project name (provide either project or project_id)
+    # @param project_id [String, nil] Project ID (UUID, provide either project or project_id)
     # @param slug [String] Prompt slug
     # @param version [String, nil] Specific version (default: latest)
     # @param defaults [Hash] Default variable values for build()
     # @param api [API, nil] Braintrust API client (default: creates one using global state)
     # @return [Prompt]
-    def self.load(project:, slug:, version: nil, defaults: {}, api: nil)
+    def self.load(slug:, project: nil, project_id: nil, version: nil, defaults: {}, api: nil)
+      raise ArgumentError, "Either project or project_id is required" unless project || project_id
+
       api ||= API.new
 
       # Find the function by project + slug
-      result = api.functions.list(project_name: project, slug: slug)
+      result = api.functions.list(project_name: project, project_id: project_id, slug: slug)
       function = result.dig("objects")&.first
-      raise Error, "Prompt '#{slug}' not found in project '#{project}'" unless function
+
+      identifier = project ? "project '#{project}'" : "project_id '#{project_id}'"
+      raise Error, "Prompt '#{slug}' not found in #{identifier}" unless function
 
       # Fetch full function data including prompt_data
       full_data = api.functions.get(id: function["id"], version: version)
@@ -47,6 +52,7 @@ module Braintrust
       @name = data["name"]
       @slug = data["slug"]
       @project_id = data["project_id"]
+      @version = data["_xact_id"]
     end
 
     # Get the raw prompt definition