braintrust 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 19c996fdd8b5b96cb52af3d3dfd855d26fc73338efffa2c6c4ada7d71fdc0e76
-  data.tar.gz: fada237f8610fee6f54aa0c820777900c31e6417312e5e0305df60744995e190
+  metadata.gz: 0f30760b63f57dfa236f8f8f74c60aabad6e693f86a57bf8699b028eb00e8639
+  data.tar.gz: fcae112dc4175b2248a853405587921f16eb2c67d2b8930e2a3877cc09b9e9d1
 SHA512:
-  metadata.gz: de4d8d52ecb56254ef041df2ae0beb085ff69a23f2cc66cf472862ded5380736fb79d6fa05b8fa3b566a3de0539a9164d81b94c37a7a4bad13d7920df2e27889
-  data.tar.gz: 1c285c2a42009decb4a0bb51b224bed27d19650fc4bb71b66c27ad074d557cb4932b07818614f5e9219f4d713d1cc3c4b9fb82756ce37a7c5e1aa12e3fbcc438
+  metadata.gz: 16e96c1f75646d2b581cb7a5c1c50ca66de3e625c75da11c6a9fd263313adea9a936e1e30f4a9733e16e56d3120737731d47ab89d858aafd7543b75011cbc9de
+  data.tar.gz: b926449904f3dafe6803f76105ee8d85b134c777b827c93b877390e318c167e1f8212eab7f207e1fc9a8e8b77cd6a2d48c5c374f412fc4af0a0f614a6c4de94e

data/lib/braintrust/api/datasets.rb

@@ -164,7 +164,7 @@ module Braintrust
           raise ArgumentError, "Unsupported HTTP method: #{method}"
         end
 
-        request["Authorization"] = "Bearer #{@state.api_key}"
+        request["Authorization"] = "Bearer #{@state.api_key!}"
 
         # Execute request with timing
         start_time = Time.now

data/lib/braintrust/api/functions.rb

@@ -239,7 +239,7 @@ module Braintrust
           raise ArgumentError, "Unsupported HTTP method: #{method}"
         end
 
-        request["Authorization"] = "Bearer #{@state.api_key}"
+        request["Authorization"] = "Bearer #{@state.api_key!}"
 
         # Execute request with timing
         start_time = Time.now

data/lib/braintrust/api/internal/btql.rb

@@ -63,7 +63,7 @@ module Braintrust
 
           request = Net::HTTP::Post.new(uri)
           request["Content-Type"] = "application/json"
-          request["Authorization"] = "Bearer #{@state.api_key}"
+          request["Authorization"] = "Bearer #{@state.api_key!}"
           request["Accept"] = "application/x-jsonlines"
           request.body = JSON.dump(payload)
 

data/lib/braintrust/api/internal/experiments.rb

@@ -39,7 +39,7 @@ module Braintrust
 
           request = Net::HTTP::Post.new(uri)
           request["Content-Type"] = "application/json"
-          request["Authorization"] = "Bearer #{@state.api_key}"
+          request["Authorization"] = "Bearer #{@state.api_key!}"
           request.body = JSON.dump(payload)
 
           response = Braintrust::Internal::Http.with_redirects(uri, request)
@@ -59,7 +59,7 @@ module Braintrust
           uri = URI("#{@state.api_url}/v1/experiment/#{id}")
 
           request = Net::HTTP::Delete.new(uri)
-          request["Authorization"] = "Bearer #{@state.api_key}"
+          request["Authorization"] = "Bearer #{@state.api_key!}"
 
           response = Braintrust::Internal::Http.with_redirects(uri, request)
 

data/lib/braintrust/api/internal/projects.rb

@@ -24,7 +24,7 @@ module Braintrust
 
           request = Net::HTTP::Post.new(uri)
           request["Content-Type"] = "application/json"
-          request["Authorization"] = "Bearer #{@state.api_key}"
+          request["Authorization"] = "Bearer #{@state.api_key!}"
           request.body = JSON.dump({name: name})
 
           response = Braintrust::Internal::Http.with_redirects(uri, request)
@@ -44,7 +44,7 @@ module Braintrust
           uri = URI("#{@state.api_url}/v1/project/#{id}")
 
           request = Net::HTTP::Delete.new(uri)
-          request["Authorization"] = "Bearer #{@state.api_key}"
+          request["Authorization"] = "Bearer #{@state.api_key!}"
 
           response = Braintrust::Internal::Http.with_redirects(uri, request)
 

data/lib/braintrust/classifier.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+require_relative "internal/callable"
+
+module Braintrust
+  # Classifier wraps a classification function that categorizes and labels eval outputs.
+  #
+  # Unlike scorers (which return numeric 0-1 values), classifiers return structured
+  # {Classification} items with an id and optional label and metadata.
+  #
+  # Use inline with a block (keyword args):
+  #   classifier = Classifier.new("category") { |output:| {name: "category", id: "greeting", label: "Greeting"} }
+  #
+  # Or include in a class and define #call with keyword args:
+  #   class CategoryClassifier
+  #     include Braintrust::Classifier
+  #
+  #     def call(output:)
+  #       {name: "category", id: "greeting", label: "Greeting"}
+  #     end
+  #   end
+  #
+  # Classifiers may return a single Classification hash, an Array of them, or nil
+  # (meaning no classifications for this case).
+  module Classifier
+    DEFAULT_NAME = "classifier"
+
+    # @param base [Class] the class including Classifier
+    def self.included(base)
+      base.include(Callable)
+    end
+
+    # Create a block-based classifier.
+    #
+    # @param name [String, nil] optional name (defaults to "classifier")
+    # @param block [Proc] the classification implementation; declare only the keyword
+    #   args you need. Extra kwargs are filtered out automatically.
+    #
+    #   Supported kwargs: +input:+, +expected:+, +output:+, +metadata:+, +trace:+, +parameters:+
+    # @return [Classifier::Block]
+    # @raise [ArgumentError] if the block has unsupported arity
+    def self.new(name = nil, &block)
+      Block.new(name: name || DEFAULT_NAME, &block)
+    end
+
+    # Included into classes that +include Classifier+. Prepends KeywordFilter and
+    # ClassificationNormalizer so #call receives only declared kwargs and always returns
+    # Array<Hash>. Also provides a default #name and #call_parameters.
+    module Callable
+      # Normalizes the raw return value of #call into Array<Hash>.
+      # Nested inside Callable because it depends on #name which Callable provides.
+      module ClassificationNormalizer
+        # @return [Array<Hash>] normalized classification hashes with :name, :id, and optional :label, :metadata keys
+        def call(**kwargs)
+          normalize_classification_result(super)
+        end
+
+        private
+
+        # @param result [Hash, Array<Hash>, nil] raw return value from #call
+        # @return [Array<Hash>] zero or more classification hashes with :name, :id keys
+        # @raise [ArgumentError] if any item is not a non-empty object
+        def normalize_classification_result(result)
+          case result
+          when nil then []
+          when Array then result.map { |item| normalize_classification_item(item) }
+          when Hash then [normalize_classification_item(result)]
+          else
+            raise ArgumentError, "When returning structured classifier results, each classification must be a non-empty object. Got: #{result.inspect}"
+          end
+        end
+
+        # Fills in missing :name from the classifier, validates :id.
+        # @param item [Hash] a classification hash
+        # @return [Hash] the item with :name defaulted and validated
+        # @raise [ArgumentError] if item is not a non-empty Hash
+        def normalize_classification_item(item)
+          unless item.is_a?(Hash) && !item.empty?
+            raise ArgumentError, "When returning structured classifier results, each classification must be a non-empty object. Got: #{item.inspect}"
+          end
+
+          # :name defaults to the classifier's resolved name when missing, empty, or non-string
+          unless item[:name].is_a?(String) && !item[:name].empty?
+            item = item.merge(name: name)
+          end
+
+          item
+        end
+      end
+
+      # Infrastructure modules prepended onto every classifier class.
+      # Used both to set up the ancestor chain and to skip past them in
+      # #call_parameters so KeywordFilter sees the real call signature.
+      PREPENDED = [Internal::Callable::KeywordFilter, ClassificationNormalizer].freeze
+
+      # @param base [Class] the class including Callable
+      def self.included(base)
+        PREPENDED.each { |mod| base.prepend(mod) }
+      end
+
+      # Default name derived from the class name (e.g. CategoryClassifier -> "category_classifier").
+      # @return [String]
+      def name
+        klass = self.class.name&.split("::")&.last
+        return Classifier::DEFAULT_NAME unless klass
+        klass.gsub(/([a-z])([A-Z])/, '\1_\2').downcase
+      end
+
+      # Provides KeywordFilter with the actual call signature of the subclass.
+      # Walks past PREPENDED modules in the ancestor chain so that user-defined
+      # #call keyword params are correctly introspected.
+      # Block overrides this to point directly at @block.parameters.
+      # @return [Array<Array>] parameter list
+      def call_parameters
+        meth = method(:call)
+        meth = meth.super_method while meth.super_method && PREPENDED.include?(meth.owner)
+        meth.parameters
+      end
+    end
+
+    # Block-based classifier. Stores a Proc and delegates #call to it.
+    # Includes Classifier so it satisfies +Classifier ===+ checks.
+    # Exposes #call_parameters so KeywordFilter can introspect the block's
+    # declared kwargs rather than Block#call's **kwargs signature.
+    class Block
+      include Classifier
+
+      # @return [String]
+      attr_reader :name
+
+      # @param name [String] classifier name
+      # @param block [Proc] classification implementation; must use keyword args or zero-arity
+      # @raise [ArgumentError] if the block uses positional params
+      def initialize(name: DEFAULT_NAME, &block)
+        @name = name
+        params = block.parameters
+        unless Internal::Callable::KeywordFilter.has_any_keywords?(params) || block.arity == 0
+          raise ArgumentError, "Classifier block must use keyword args (got arity #{block.arity})"
+        end
+        @block = block
+      end
+
+      # @param kwargs [Hash] keyword arguments (filtered by KeywordFilter)
+      # @return [Array<Hash>] normalized classification results
+      def call(**kwargs)
+        @block.call(**kwargs)
+      end
+
+      # Exposes the block's parameter list so KeywordFilter can filter
+      # kwargs to match the block's declared keywords.
+      # @return [Array<Array>] parameter list from Proc#parameters
+      def call_parameters
+        @block.parameters
+      end
+    end
+  end
+end

data/lib/braintrust/config.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "internal/api_key_resolver"
+
 module Braintrust
   # Configuration object that reads from environment variables
   # and allows overriding with explicit options
@@ -39,7 +41,7 @@ module Braintrust
       end
 
       new(
-        api_key: api_key || ((ENV["BRAINTRUST_API_KEY"] && ENV["BRAINTRUST_API_KEY"].empty?) ? nil : ENV["BRAINTRUST_API_KEY"]),
+        api_key: Internal::ApiKeyResolver.resolve(explicit_api_key: api_key),
         org_name: org_name || ENV["BRAINTRUST_ORG_NAME"],
         default_project: default_project || ENV["BRAINTRUST_DEFAULT_PROJECT"],
         app_url: app_url || ENV["BRAINTRUST_APP_URL"] || "https://www.braintrust.dev",

data/lib/braintrust/eval/context.rb

@@ -1,18 +1,20 @@
 # frozen_string_literal: true
 
 require_relative "cases"
+require_relative "../classifier"
 
 module Braintrust
   module Eval
     # Holds all normalized, ready-to-execute eval components.
     # Use Context.build to construct from raw user inputs.
     class Context
-      attr_reader :task, :scorers, :cases, :experiment_id, :experiment_name,
-        :project_id, :project_name, :state, :tracer_provider,
+      attr_reader :task, :scorers, :classifiers, :cases, :experiment_id,
+        :experiment_name, :project_id, :project_name, :state, :tracer_provider,
         :on_progress, :parent_span_attr, :generation, :parameters
 
       # @param task [Task] Normalized task wrapper
       # @param scorers [Array<Scorer>] Normalized scorer wrappers
+      # @param classifiers [Array<Classifier>] Normalized classifier wrappers
       # @param cases [Cases] Normalized eval cases
       # @param experiment_id [String, nil] Experiment ID for logging and trace linkage
       # @param experiment_name [String, nil] Experiment name, included in span attributes
@@ -24,11 +26,13 @@ module Braintrust
       # @param parent_span_attr [String, nil] Formatted parent span identifier ("type:id"), linking spans to a parent context
       # @param generation [Integer, nil] Generation number from the parent span context, used to link spans in a trace hierarchy
       # @param parameters [Hash, nil] Runtime parameters passed to task and scorers as a `parameters:` keyword argument
-      def initialize(task:, scorers:, cases:, experiment_id: nil, experiment_name: nil,
-        project_id: nil, project_name: nil, state: nil, tracer_provider: nil,
-        on_progress: nil, parent_span_attr: nil, generation: nil, parameters: nil)
+      def initialize(task:, scorers:, cases:, classifiers: [],
+        experiment_id: nil, experiment_name: nil, project_id: nil,
+        project_name: nil, state: nil, tracer_provider: nil, on_progress: nil,
+        parent_span_attr: nil, generation: nil, parameters: nil)
         @task = task
         @scorers = scorers
+        @classifiers = classifiers
         @cases = cases
         @experiment_id = experiment_id
         @experiment_name = experiment_name
@@ -46,6 +50,7 @@ module Braintrust
       # Delegates to Factory for normalization.
       # @param task [Task, Proc, #call] Task to evaluate; wrapped into a {Task} if needed
       # @param scorers [Array<Scorer, Proc, String, Scorer::ID, #call>] Scorers; each is normalized into a {Scorer}
+      # @param classifiers [Array<Classifier, Proc, #call>] Classifiers; each is normalized into a {Classifier}
       # @param cases [Cases, Array, Enumerable] Eval cases; wrapped into {Cases} if needed
       # @param experiment_id [String, nil] Experiment ID for logging
       # @param experiment_name [String, nil] Experiment name, included in span attributes
@@ -57,14 +62,15 @@ module Braintrust
       # @param parent [Hash, nil] Parent span info with keys :object_type, :object_id, and optionally :generation
       # @param parameters [Hash, nil] Runtime parameters passed to task and scorers as a `parameters:` keyword argument
       # @return [Context]
-      def self.build(task:, scorers:, cases:, experiment_id: nil, experiment_name: nil,
-        project_id: nil, project_name: nil, state: nil, tracer_provider: nil,
-        on_progress: nil, parent: nil, parameters: nil)
+      def self.build(task:, scorers:, cases:, classifiers: [],
+        experiment_id: nil, experiment_name: nil, project_id: nil,
+        project_name: nil, state: nil, tracer_provider: nil, on_progress: nil,
+        parent: nil, parameters: nil)
         Factory.new(
           state: state, tracer_provider: tracer_provider,
           project_id: project_id, project_name: project_name
         ).build(
-          task: task, scorers: scorers, cases: cases,
+          task: task, scorers: scorers, classifiers: classifiers, cases: cases,
           experiment_id: experiment_id, experiment_name: experiment_name,
           on_progress: on_progress, parent: parent, parameters: parameters
         )
@@ -86,17 +92,19 @@ module Braintrust
         # Normalize raw inputs and construct a {Context}.
         # @param task [Task, Proc, #call] Raw task
         # @param scorers [Array] Raw scorers
+        # @param classifiers [Array] Raw classifiers
         # @param cases [Cases, Array, Enumerable] Raw eval cases
         # @param experiment_id [String, nil]
         # @param experiment_name [String, nil]
         # @param on_progress [Proc, nil]
         # @param parent [Hash, nil] Parent span info with keys :object_type, :object_id, and optionally :generation
         # @return [Context]
-        def build(task:, scorers:, cases:, experiment_id: nil, experiment_name: nil,
-          on_progress: nil, parent: nil, parameters: nil)
+        def build(task:, scorers:, cases:, classifiers: [], experiment_id: nil,
+          experiment_name: nil, on_progress: nil, parent: nil, parameters: nil)
           Context.new(
             task: normalize_task(task),
             scorers: normalize_scorers(scorers),
+            classifiers: normalize_classifiers(classifiers),
             cases: normalize_cases(cases),
             experiment_id: experiment_id,
             experiment_name: experiment_name,
@@ -188,6 +196,23 @@ module Braintrust
             end
           end
         end
+
+        # @param raw [Array<Classifier, Proc, #call>]
+        # @return [Array<Classifier>]
+        def normalize_classifiers(raw)
+          raw.map do |classifier|
+            case classifier
+            when Braintrust::Classifier
+              classifier
+            when Proc
+              # Pass Proc/Lambda directly to preserve keyword arg info
+              Braintrust::Classifier.new(&classifier)
+            else
+              name = classifier.respond_to?(:name) ? classifier.name : nil
+              Braintrust::Classifier.new(name, &classifier.method(:call))
+            end
+          end
+        end
       end
     end
   end

data/lib/braintrust/eval/evaluator.rb

@@ -40,11 +40,12 @@ module Braintrust
     #     }
     #   )
     class Evaluator
-      attr_accessor :task, :scorers, :parameters
+      attr_accessor :task, :scorers, :classifiers, :parameters
 
-      def initialize(task: nil, scorers: [], parameters: {})
+      def initialize(task: nil, scorers: [], classifiers: [], parameters: {})
         @task = task
         @scorers = scorers
+        @classifiers = classifiers
         @parameters = parameters
       end
 
@@ -68,6 +69,7 @@ module Braintrust
       # @param project_id [String, nil] Project UUID (skips project creation)
       # @param dataset [String, Hash, Dataset, Dataset::ID, nil] Dataset to fetch
       # @param scorers [Array, nil] Additional scorers (merged with evaluator's own)
+      # @param classifiers [Array, nil] Additional classifiers (merged with evaluator's own)
       # @param parent [Hash, nil] Parent span context
       # @param state [State, nil] Braintrust state
       # @param update [Boolean] If true, allow reusing existing experiment (default: false)
@@ -75,16 +77,19 @@ module Braintrust
       # @return [Result]
       def run(cases, on_progress: nil, quiet: false,
         project: nil, experiment: nil, project_id: nil,
-        dataset: nil, scorers: nil, parent: nil,
+        dataset: nil, scorers: nil, classifiers: nil, parent: nil,
         state: nil, update: false, tracer_provider: nil,
         parameters: nil)
         all_scorers = scorers ? self.scorers + scorers : self.scorers
+        all_classifiers = classifiers ?
+          self.classifiers + classifiers :
+          self.classifiers
         Braintrust::Eval.run(
-          task: task, scorers: all_scorers, cases: cases, dataset: dataset,
-          project: project, experiment: experiment, project_id: project_id,
-          parent: parent, on_progress: on_progress, quiet: quiet,
-          state: state, update: update, tracer_provider: tracer_provider,
-          parameters: parameters
+          task: task, scorers: all_scorers, classifiers: all_classifiers,
+          cases: cases, dataset: dataset, project: project,
+          experiment: experiment, project_id: project_id, parent: parent,
+          on_progress: on_progress, quiet: quiet, state: state, update: update,
+          tracer_provider: tracer_provider, parameters: parameters
         )
       end
     end

data/lib/braintrust/eval/result.rb

@@ -9,7 +9,7 @@ module Braintrust
     # Contains experiment metadata, errors, timing information, and raw score data
     class Result
       attr_reader :experiment_id, :experiment_name, :project_id, :project_name,
-        :permalink, :errors, :duration, :scores
+        :permalink, :errors, :duration, :scores, :classifications
 
       # Create a new result
       # @param experiment_id [String] The experiment ID
@@ -20,8 +20,9 @@ module Braintrust
       # @param errors [Array<String>] List of errors that occurred
       # @param duration [Float] Duration in seconds
       # @param scores [Hash, nil] Raw score data { scorer_name => Array<Numeric> }
+      # @param classifications [Hash, nil] Classification results { name => Array<ClassificationItem> }, nil when no classifiers ran
       def initialize(experiment_id:, experiment_name:, project_id:, project_name:,
-        permalink:, errors:, duration:, scores: nil)
+        permalink:, errors:, duration:, scores: nil, classifications: nil)
         @experiment_id = experiment_id
         @experiment_name = experiment_name
         @project_id = project_id
@@ -30,6 +31,7 @@ module Braintrust
         @errors = errors
         @duration = duration
         @scores = scores
+        @classifications = classifications
       end
 
       # Check if the evaluation was successful (no errors)

data/lib/braintrust/eval/runner.rb

@@ -27,8 +27,9 @@ module Braintrust
         @eval_context = eval_context
         @tracer = eval_context.tracer_provider.tracer("braintrust-eval")
 
-        # Mutex for thread-safe score collection
+        # Mutexes for thread-safe result collection
         @score_mutex = Mutex.new
+        @classification_mutex = Mutex.new
       end
 
       # Run evaluation and return Result
@@ -39,6 +40,7 @@ module Braintrust
         eval_cases = eval_context.cases
         errors = Queue.new
         @scores = {} # Reset for each run: { scorer_name => Array<Numeric> }
+        @classifications = {} # Reset for each run: { classifier_name => Array<ClassificationItem> }
 
         if parallelism && parallelism > 1
           Internal::ThreadPool.each(eval_cases, parallelism: parallelism) do |eval_case|
@@ -69,7 +71,8 @@ module Braintrust
           permalink: permalink,
           errors: error_array,
           duration: duration,
-          scores: @scores
+          scores: @scores,
+          classifications: @classifications.empty? ? nil : @classifications
         )
       end
 
@@ -119,6 +122,17 @@ module Braintrust
             errors << "Scorers failed for input '#{kase.input}': #{e.message}"
           end
 
+          # Run classifiers (independent of scorers; errors do not abort eval)
+          classifier_errors = run_classifiers(kase, eval_span)
+          unless classifier_errors.empty?
+            existing_metadata = kase.metadata || {}
+            classifier_errors_metadata = existing_metadata.merge(classifier_errors: classifier_errors)
+            set_json_attr(eval_span, "braintrust.metadata", classifier_errors_metadata)
+            classifier_errors.each do |classifier_name, message|
+              errors << "Classifier '#{classifier_name}' failed for input '#{kase.input}': #{message}"
+            end
+          end
+
           # Set output after task completes
           set_json_attr(eval_span, "braintrust.output_json", {output: kase.output})
 
@@ -318,6 +332,104 @@ module Braintrust
           score_results.each { |s| (@scores[s[:name]] ||= []) << s[:score] }
         end
       end
+
+      # Run all classifiers for a case. Classifier errors are non-fatal and stored in metadata.
+      # @param kase [CaseContext] The per-case context (output must be populated)
+      # @param eval_span [OpenTelemetry::Trace::Span] The eval span for this case
+      # @return [Hash] classifier_errors map (name -> error message), empty if no errors
+      def run_classifiers(kase, eval_span)
+        return {} if eval_context.classifiers.empty?
+
+        classifier_kwargs = {
+          input: kase.input,
+          expected: kase.expected,
+          output: kase.output,
+          metadata: kase.metadata || {},
+          trace: kase.trace,
+          parameters: eval_context.parameters || {}
+        }
+        classifier_input = {
+          input: kase.input,
+          expected: kase.expected,
+          output: kase.output,
+          metadata: kase.metadata || {},
+          parameters: eval_context.parameters || {}
+        }
+
+        case_classifications = {}
+        classifier_errors = {}
+
+        eval_context.classifiers.each_with_index do |classifier, index|
+          classifier_name = classifier.name || "classifier_#{index}"
+          begin
+            results = run_classifier(classifier, classifier_kwargs, classifier_input)
+            results.each do |item|
+              item_name = item[:name]
+              classification_item = item.except(:name)
+              (case_classifications[item_name] ||= []) << classification_item
+            end
+            collect_classifications(results)
+          rescue => e
+            Braintrust::Log.warn("[Classifier] #{classifier_name} failed: #{e.message}")
+            classifier_errors[classifier_name] = e.message
+          end
+        end
+
+        unless case_classifications.empty?
+          set_json_attr(eval_span, "braintrust.classifications", case_classifications)
+        end
+
+        classifier_errors
+      end
+
+      # Run a single classifier inside its own span.
+      # @param classifier [Classifier] The classifier to run
+      # @param classifier_kwargs [Hash] Keyword arguments for the classifier
+      # @param classifier_input [Hash] Input to log on the span
+      # @return [Array<Hash>] Normalized classification results from the classifier
+      def run_classifier(classifier, classifier_kwargs, classifier_input)
+        tracer.in_span(classifier.name) do |classifier_span|
+          classifier_span.set_attribute("braintrust.parent", eval_context.parent_span_attr) if eval_context.parent_span_attr
+          set_json_attr(classifier_span, "braintrust.span_attributes", build_classifier_span_attributes(classifier.name))
+          set_json_attr(classifier_span, "braintrust.input_json", classifier_input)
+
+          classification_results = classifier.call(**classifier_kwargs)
+
+          # Build output dict keyed by name -> array of items (for span logging)
+          output_by_name = {}
+          classification_results.each do |item|
+            (output_by_name[item[:name]] ||= []) << item.except(:name)
+          end
+
+          set_json_attr(classifier_span, "braintrust.output_json", output_by_name)
+
+          classification_results
+        rescue => e
+          record_span_error(classifier_span, e, "ClassifierError")
+          raise
+        end
+      end
+
+      # Build span_attributes for a classifier span.
+      # @param classifier_name [String] The classifier name
+      # @return [Hash]
+      def build_classifier_span_attributes(classifier_name)
+        attrs = {type: "classifier", name: classifier_name, purpose: "scorer"}
+        attrs[:generation] = eval_context.generation if eval_context.generation
+        attrs
+      end
+
+      # Collect classification results into the global accumulator (thread-safe).
+      # Converts Classification to ClassificationItem by dropping :name.
+      # @param classification_results [Array<Hash>] Classification results from a classifier
+      def collect_classifications(classification_results)
+        @classification_mutex.synchronize do
+          classification_results.each do |item|
+            item_name = item[:name]
+            (@classifications[item_name] ||= []) << item.except(:name)
+          end
+        end
+      end
     end
   end
 end

data/lib/braintrust/eval.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "classifier"
 require_relative "scorer"
 require_relative "task"
 require_relative "functions"
@@ -160,7 +161,10 @@ module Braintrust
       #   - String: dataset name (fetches from same project)
       #   - Hash: {name:, id:, project:, version:, limit:}
       # @param task [#call] The task to evaluate (must be callable)
-      # @param scorers [Array<String, Scorer, #call>] The scorers to use (String names, Scorer objects, or callables)
+      # @param scorers [Array<String, Scorer, #call>, nil] The scorers to use (String names, Scorer objects, or callables).
+      #   At least one of scorers or classifiers must be provided.
+      # @param classifiers [Array<Classifier, #call>, nil] The classifiers to use.
+      #   At least one of scorers or classifiers must be provided.
       # @param on_progress [#call, nil] Optional callback fired after each test case.
       #   Receives a Hash: {"data" => output, "scores" => {name => value}} on success,
       #   or {"error" => message} on failure.
@@ -177,13 +181,16 @@ module Braintrust
       # @param parent [Hash, nil] Parent span context ({object_type:, object_id:, generation:})
       # @param parameters [Hash, nil] Runtime parameters passed to task and scorers as a `parameters:` keyword argument
       # @return [Result]
-      def run(task:, scorers:, project: nil, experiment: nil,
-        cases: nil, dataset: nil, on_progress: nil,
+      def run(task:, scorers: nil, classifiers: nil, project: nil,
+        experiment: nil, cases: nil, dataset: nil, on_progress: nil,
         parallelism: 1, tags: nil, metadata: nil, update: false, quiet: false,
         state: nil, tracer_provider: nil, project_id: nil, parent: nil,
         parameters: nil)
         # Validate required parameters
-        validate_params!(task: task, scorers: scorers, cases: cases, dataset: dataset)
+        validate_params!(task: task, scorers: scorers,
+          classifiers: classifiers, cases: cases, dataset: dataset)
+        scorers ||= []
+        classifiers ||= []
 
         experiment_id = nil
         project_name = project
@@ -216,6 +223,7 @@ module Braintrust
         context = Context.build(
           task: task,
           scorers: scorers,
+          classifiers: classifiers,
           cases: cases,
           experiment_id: experiment_id,
           experiment_name: experiment,
@@ -245,9 +253,19 @@ module Braintrust
 
       # Validate required parameters
       # @raise [ArgumentError] if validation fails
-      def validate_params!(task:, scorers:, cases:, dataset:)
+      def validate_params!(task:, scorers:, classifiers:, cases:, dataset:)
         raise ArgumentError, "task is required" unless task
-        raise ArgumentError, "scorers is required" unless scorers
+
+        # Validate task is callable before anything else
+        unless task.respond_to?(:call)
+          raise ArgumentError, "task must be callable (respond to :call)"
+        end
+
+        has_scorers = scorers && !scorers.empty?
+        has_classifiers = classifiers && !classifiers.empty?
+        unless has_scorers || has_classifiers
+          raise ArgumentError, "at least one of scorers or classifiers is required"
+        end
 
         # Validate cases and dataset are mutually exclusive
         if cases && dataset
@@ -258,11 +276,6 @@ module Braintrust
         unless cases || dataset
           raise ArgumentError, "must specify either 'cases' or 'dataset'"
         end
-
-        # Validate task is callable
-        unless task.respond_to?(:call)
-          raise ArgumentError, "task must be callable (respond to :call)"
-        end
       end
 
       # Resolve project by name or ID. Creates if needed.

data/lib/braintrust/internal/api_key_resolver.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Braintrust
+  module Internal
+    # Resolves the Braintrust API key from explicit options, ENV, or the nearest
+    # .braintrust.json file without mutating the process environment.
+    class ApiKeyResolver
+      ENV_KEY = "BRAINTRUST_API_KEY"
+      CONFIG_FILE = ".braintrust.json"
+      SEARCH_PARENT_LIMIT = 64
+
+      def self.resolve(explicit_api_key: nil, start_dir: Dir.pwd)
+        return explicit_api_key unless explicit_api_key.nil?
+
+        env_api_key = ENV[ENV_KEY]
+        return env_api_key if env_api_key && !env_api_key.strip.empty?
+
+        find_file_api_key(start_dir)
+      end
+
+      def self.find_file_api_key(start_dir = Dir.pwd)
+        dir = start_dir
+
+        0.upto(SEARCH_PARENT_LIMIT) do
+          config_path = File.join(dir, CONFIG_FILE)
+
+          begin
+            contents = File.read(config_path)
+          rescue Errno::ENOENT, Errno::ENOTDIR
+            # Missing candidates are not boundaries; keep walking upward.
+          rescue
+            return nil
+          else
+            return parse_api_key(contents)
+          end
+
+          parent = File.dirname(dir)
+          break if parent == dir
+          dir = parent
+        end
+
+        nil
+      rescue
+        nil
+      end
+
+      def self.parse_api_key(contents)
+        config = JSON.parse(contents)
+        return nil unless config.is_a?(Hash)
+
+        value = config[ENV_KEY]
+        (value.is_a?(String) && !value.strip.empty?) ? value : nil
+      rescue JSON::ParserError, TypeError
+        nil
+      end
+
+      private_class_method :find_file_api_key, :parse_api_key
+    end
+  end
+end

data/lib/braintrust/server/services/list_service.rb

@@ -20,6 +20,11 @@ module Braintrust
               {"name" => scorer_name}
             end
             entry = {"scores" => scores}
+            classifiers = (evaluator.classifiers || []).each_with_index.map do |classifier, i|
+              classifier_name = classifier.respond_to?(:name) ? classifier.name : "classifier_#{i}"
+              {"name" => classifier_name}
+            end
+            entry["classifiers"] = classifiers unless classifiers.empty?
             params = serialize_parameters(evaluator.parameters)
             entry["parameters"] = params if params
             result[name] = entry

data/lib/braintrust/setup.rb

@@ -11,7 +11,7 @@
 #   require "braintrust/setup"
 #
 # Environment variables:
-#   BRAINTRUST_API_KEY - Required for tracing to work
+#   BRAINTRUST_API_KEY - Required for tracing to work; falls back to .braintrust.json
 #   BRAINTRUST_AUTO_INSTRUMENT - Set to "false" to disable (default: true)
 #   BRAINTRUST_INSTRUMENT_ONLY - Comma-separated whitelist
 #   BRAINTRUST_INSTRUMENT_EXCEPT - Comma-separated blacklist

data/lib/braintrust/state.rb

@@ -6,6 +6,8 @@ module Braintrust
   # State object that holds Braintrust configuration
   # Thread-safe global state management
   class State
+    class MissingAPIKeyError < ArgumentError; end
+
     attr_reader :api_key, :org_name, :org_id, :default_project, :app_url, :api_url, :proxy_url, :logged_in, :config
 
     @mutex = Mutex.new
@@ -66,7 +68,7 @@ module Braintrust
     def initialize(api_key: nil, org_name: nil, org_id: nil, default_project: nil, app_url: nil, api_url: nil, proxy_url: nil, blocking_login: false, enable_tracing: true, tracer_provider: nil, config: nil, exporter: nil)
       # Instance-level mutex for thread-safe login
       @login_mutex = Mutex.new
-      raise ArgumentError, "api_key is required" if api_key.nil? || api_key.empty?
+      raise MissingAPIKeyError, "api_key is required" if api_key.nil? || api_key.empty?
 
       @api_key = api_key
       @org_name = org_name
@@ -101,6 +103,11 @@ module Braintrust
       end
     end
 
+    def api_key!
+      raise MissingAPIKeyError, "api_key is required" if @api_key.nil? || @api_key.empty?
+      @api_key
+    end
+
     # Thread-safe global state getter
     def self.global
       @mutex.synchronize { @global_state }
@@ -121,9 +128,10 @@ module Braintrust
       @login_mutex.synchronize do
         # Return early if already logged in
         return self if @logged_in
+        api_key = api_key!
 
         result = API::Internal::Auth.login(
-          api_key: @api_key,
+          api_key: api_key,
           app_url: @app_url,
           org_name: @org_name
         )
@@ -167,6 +175,9 @@ module Braintrust
           login
           Log.debug("Background login succeeded")
           break
+        rescue MissingAPIKeyError => e
+          Log.debug("Background login skipped: #{e.message}")
+          break
         rescue => e
           retry_count += 1
           delay = [0.001 * 2**(retry_count - 1), max_delay].min
@@ -190,7 +201,7 @@ module Braintrust
     # Raises ArgumentError if state is invalid
     # @return [self]
     def validate
-      raise ArgumentError, "api_key is required" if @api_key.nil? || @api_key.empty?
+      api_key!
       raise ArgumentError, "api_url is required" if @api_url.nil? || @api_url.empty?
       raise ArgumentError, "app_url is required" if @app_url.nil? || @app_url.empty?
 

data/lib/braintrust/trace/span_exporter.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "opentelemetry/exporter/otlp"
+require_relative "../state"
 
 module Braintrust
   module Trace
@@ -18,6 +19,8 @@ module Braintrust
       FAILURE = OpenTelemetry::SDK::Trace::Export::FAILURE
 
       def initialize(endpoint:, api_key:)
+        raise State::MissingAPIKeyError, "api_key is required" if api_key.nil? || api_key.empty?
+
         super(endpoint: endpoint, headers: {"Authorization" => "Bearer #{api_key}"})
       end
 

data/lib/braintrust/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Braintrust
-  VERSION = "0.3.2"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: braintrust
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - Braintrust
@@ -90,6 +90,7 @@ files:
 - lib/braintrust/api/internal/btql.rb
 - lib/braintrust/api/internal/experiments.rb
 - lib/braintrust/api/internal/projects.rb
+- lib/braintrust/classifier.rb
 - lib/braintrust/config.rb
 - lib/braintrust/contrib.rb
 - lib/braintrust/contrib/anthropic/deprecated.rb
@@ -147,6 +148,7 @@ files:
 - lib/braintrust/eval/summary.rb
 - lib/braintrust/eval/trace.rb
 - lib/braintrust/functions.rb
+- lib/braintrust/internal/api_key_resolver.rb
 - lib/braintrust/internal/callable.rb
 - lib/braintrust/internal/encoding.rb
 - lib/braintrust/internal/env.rb
@@ -213,7 +215,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Ruby SDK for Braintrust
 test_files: []