braintrust 0.2.0 → 0.2.1

data/lib/braintrust/internal/callable.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Internal
+    module Callable
+      # Filters keyword arguments so callers can pass a superset of kwargs
+      # and the receiver only gets the ones it declared. This avoids Ruby 3.2+
+      # ArgumentError for unknown keywords without requiring ** on every definition.
+      #
+      # When prepended on a class, intercepts #call and slices kwargs to match
+      # the declared parameters before forwarding. Methods with **keyrest
+      # receive all kwargs unfiltered.
+      #
+      # @example
+      #   class Greeter
+      #     prepend Internal::Callable::KeywordFilter
+      #     def call(name:)
+      #       "hello #{name}"
+      #     end
+      #   end
+      #   Greeter.new.call(name: "world", extra: "ignored")  # => "hello world"
+      module KeywordFilter
+        # Filter kwargs to only the keyword params declared by the given parameters list.
+        # Returns kwargs unchanged if parameters include **keyrest.
+        #
+        # @param params [Array<Array>] parameter list from Proc#parameters or Method#parameters
+        # @param kwargs [Hash] keyword arguments to filter
+        # @return [Hash] filtered keyword arguments
+        def self.filter(params, kwargs)
+          return kwargs if has_keyword_splat?(params)
+
+          declared_keys = params
+            .select { |type, _| type == :keyreq || type == :key }
+            .map(&:last)
+          kwargs.slice(*declared_keys)
+        end
+
+        # Wrap a Proc to filter kwargs to only its declared keyword params.
+        # Returns the block unchanged if it accepts **keyrest.
+        #
+        # @param block [Proc] the block to wrap
+        # @return [Proc] a wrapper that filters kwargs, or the original block
+        def self.wrap_block(block)
+          return block if has_keyword_splat?(block.parameters)
+          ->(**kw) { block.call(**filter(block.parameters, kw)) }
+        end
+
+        # Whether params include ** (keyword splat / keyrest).
+        #
+        # @param params [Array<Array>] parameter list
+        # @return [Boolean]
+        def self.has_keyword_splat?(params)
+          params.any? { |type, _| type == :keyrest }
+        end
+
+        # Whether params include any keyword parameters (key, keyreq, or keyrest).
+        #
+        # @param params [Array<Array>] parameter list
+        # @return [Boolean]
+        def self.has_any_keywords?(params)
+          params.any? { |type, _| type == :keyreq || type == :key || type == :keyrest }
+        end
+
+        # When prepended, filters kwargs before the next #call in the ancestor chain.
+        # If the instance defines #call_parameters, uses those.
+        # Otherwise introspects super_method.
+        #
+        # @param kwargs [Hash] keyword arguments
+        # @return [Object] result of the filtered #call
+        def call(**kwargs)
+          params = if respond_to?(:call_parameters)
+            call_parameters
+          else
+            impl = method(:call).super_method
+            return super unless impl
+            impl.parameters
+          end
+          super(**KeywordFilter.filter(params, kwargs))
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/logger.rb

@@ -8,6 +8,7 @@ module Braintrust
     # Default to WARN unless BRAINTRUST_DEBUG is set
     level = ENV["BRAINTRUST_DEBUG"] ? Logger::DEBUG : Logger::WARN
     @logger = Logger.new($stderr, level: level)
+    @warned = Set.new
 
     class << self
       attr_accessor :logger
@@ -24,6 +25,14 @@ module Braintrust
         @logger.warn(message)
       end
 
+      # Emit a warning only once per unique key.
+      # Subsequent calls with the same key are silently ignored.
+      def warn_once(key, message)
+        return if @warned.include?(key)
+        @warned.add(key)
+        @logger.warn(message)
+      end
+
       def error(message)
         @logger.error(message)
       end

data/lib/braintrust/scorer.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require_relative "internal/callable"
+
+module Braintrust
+  # Scorer wraps a scoring function that evaluates task output against expected values.
+  #
+  # Use inline with a block (keyword args):
+  #   scorer = Scorer.new("my_scorer") { |expected:, output:| output == expected ? 1.0 : 0.0 }
+  #
+  # Or include in a class and define #call with keyword args:
+  #   class FuzzyMatch
+  #     include Braintrust::Scorer
+  #
+  #     def call(expected:, output:)
+  #       output == expected ? 1.0 : 0.0
+  #     end
+  #   end
+  #
+  # Legacy callables with 3 or 4 positional params are auto-wrapped for
+  # backwards compatibility but emit a deprecation warning.
+  module Scorer
+    DEFAULT_NAME = "scorer"
+
+    # @param base [Class] the class including Scorer
+    def self.included(base)
+      base.include(Callable)
+    end
+
+    # Create a block-based scorer.
+    #
+    # @param name [String, nil] optional name (defaults to "scorer")
+    # @param block [Proc] the scoring implementation; declare only the keyword
+    #   args you need. Extra kwargs are filtered out automatically.
+    #
+    #   Supported kwargs: +input:+, +expected:+, +output:+, +metadata:+, +trace:+
+    # @return [Scorer::Block]
+    # @raise [ArgumentError] if the block has unsupported arity
+    def self.new(name = nil, &block)
+      Block.new(name: name || DEFAULT_NAME, &block)
+    end
+
+    # Included into classes that +include Scorer+. Prepends KeywordFilter
+    # so #call receives only its declared kwargs, and provides a default #name.
+    module Callable
+      # @param base [Class] the class including Callable
+      def self.included(base)
+        base.prepend(Internal::Callable::KeywordFilter)
+      end
+
+      # Default name derived from the class name (e.g. FuzzyMatch -> "fuzzy_match").
+      # @return [String]
+      def name
+        klass = self.class.name&.split("::")&.last
+        return Scorer::DEFAULT_NAME unless klass
+        klass.gsub(/([a-z])([A-Z])/, '\1_\2').downcase
+      end
+    end
+
+    # Block-based scorer. Stores a Proc and delegates #call to it.
+    # Includes Scorer so it satisfies +Scorer ===+ checks (e.g. in Context::Factory).
+    # Exposes #call_parameters so KeywordFilter can introspect the block's
+    # declared kwargs rather than Block#call's **kwargs signature.
+    class Block
+      include Scorer
+
+      # @return [String]
+      attr_reader :name
+
+      # @param name [String] scorer name
+      # @param block [Proc] scoring implementation
+      def initialize(name: DEFAULT_NAME, &block)
+        @name = name
+        @block = wrap_block(block)
+      end
+
+      # @param kwargs [Hash] keyword arguments (filtered by KeywordFilter)
+      # @return [Float, Hash, Array] score result
+      def call(**kwargs)
+        @block.call(**kwargs)
+      end
+
+      # Exposes the block's parameter list so KeywordFilter can filter
+      # kwargs to match the block's declared keywords.
+      # @return [Array<Array>] parameter list from Proc#parameters
+      def call_parameters
+        @block.parameters
+      end
+
+      private
+
+      # Legacy positional wrapping: arity 3/4/-4/-1 maps to (input, expected, output[, metadata]).
+      # Keyword and zero-arity blocks are stored raw; KeywordFilter handles filtering at call time.
+      # @param block [Proc]
+      # @return [Proc]
+      def wrap_block(block)
+        params = block.parameters
+        if Internal::Callable::KeywordFilter.has_any_keywords?(params) || block.arity == 0
+          block
+        else
+          case block.arity
+          when 3
+            Log.warn_once(:scorer_positional_3, "Scorer with positional params (input, expected, output) is deprecated. Use keyword args: |input:, expected:, output:| instead.")
+            ->(**kw) { block.call(kw[:input], kw[:expected], kw[:output]) }
+          when 4, -4, -1
+            Log.warn_once(:scorer_positional_4, "Scorer with positional params (input, expected, output, metadata) is deprecated. Use keyword args: |input:, expected:, output:, metadata:| instead.")
+            ->(**kw) { block.call(kw[:input], kw[:expected], kw[:output], kw[:metadata]) }
+          else
+            raise ArgumentError, "Scorer must accept keyword args or 3-4 positional params (got arity #{block.arity})"
+          end
+        end
+      end
+    end
+
+    # Value object wrapping a remote scorer function UUID.
+    # Used by Eval.run to distinguish remote scorers from local callables.
+    ID = Struct.new(:function_id, :version, keyword_init: true)
+  end
+
+  # @deprecated Use {Braintrust::Scorer::ID} instead.
+  ScorerId = Scorer::ID
+end

data/lib/braintrust/server/handlers/eval.rb

@@ -124,7 +124,7 @@ module Braintrust
             end
             [cases, nil]
           elsif data.key?("dataset_id")
-            [nil, Braintrust::DatasetId.new(id: data["dataset_id"])]
+            [nil, Braintrust::Dataset::ID.new(id: data["dataset_id"])]
           elsif data.key?("dataset_name")
             dataset_opts = {name: data["dataset_name"]}
             dataset_opts[:project] = data["project_name"] if data["project_name"]
@@ -134,14 +134,14 @@ module Braintrust
           end
         end
 
-        # Map request scores array to ScorerId structs.
+        # Map request scores array to Scorer::ID structs.
         # The UI sends function_id as a nested object: {"function_id": "uuid"}.
         def resolve_remote_scorers(scores)
           return nil if scores.nil? || scores.empty?
           scores.map do |s|
             func_id = s["function_id"]
             func_id = func_id["function_id"] if func_id.is_a?(Hash)
-            Braintrust::ScorerId.new(
+            Braintrust::Scorer::ID.new(
               function_id: func_id,
               version: s["version"]
             )

data/lib/braintrust/task.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require_relative "internal/callable"
+
+module Braintrust
+  # Task wraps a callable that processes inputs.
+  #
+  # Use inline with a block (keyword args):
+  #   task = Task.new("my_task") { |input:| process(input) }
+  #
+  # Or include in a class and define #call with keyword args:
+  #   class MyTask
+  #     include Braintrust::Task
+  #
+  #     def call(input:)
+  #       process(input)
+  #     end
+  #   end
+  #
+  # Legacy callables with 1 positional param are auto-wrapped for
+  # backwards compatibility but emit a deprecation warning.
+  module Task
+    DEFAULT_NAME = "task"
+
+    # @param base [Class] the class including Task
+    def self.included(base)
+      base.include(Callable)
+    end
+
+    # Create a block-based task.
+    #
+    # @param name [String, nil] optional name (defaults to "task")
+    # @param block [Proc] the task implementation; declare only the keyword
+    #   args you need (e.g. +|input:|+). Extra kwargs passed by the caller
+    #   are filtered out automatically.
+    # @return [Task::Block]
+    # @raise [ArgumentError] if the block has unsupported arity
+    def self.new(name = nil, &block)
+      Block.new(name: name || DEFAULT_NAME, &block)
+    end
+
+    # Included into classes that +include Task+. Prepends KeywordFilter
+    # so #call receives only its declared kwargs, and provides a default #name.
+    module Callable
+      # @param base [Class] the class including Callable
+      def self.included(base)
+        base.prepend(Internal::Callable::KeywordFilter)
+      end
+
+      # Default name derived from the class name (e.g. MyTask -> "my_task").
+      # @return [String]
+      def name
+        klass = self.class.name&.split("::")&.last
+        return Task::DEFAULT_NAME unless klass
+        klass.gsub(/([a-z])([A-Z])/, '\1_\2').downcase
+      end
+    end
+
+    # Block-based task. Stores a Proc and delegates #call to it.
+    # Includes Task so it satisfies +Task ===+ checks (e.g. in Context::Factory).
+    # Exposes #call_parameters so KeywordFilter can introspect the block's
+    # declared kwargs rather than Block#call's **kwargs signature.
+    class Block
+      include Task
+
+      # @return [String]
+      attr_reader :name
+
+      # @param name [String] task name
+      # @param block [Proc] task implementation
+      def initialize(name: DEFAULT_NAME, &block)
+        @name = name
+        @block = wrap_block(block)
+      end
+
+      # @param kwargs [Hash] keyword arguments (filtered by KeywordFilter)
+      # @return [Object] result of the block
+      def call(**kwargs)
+        @block.call(**kwargs)
+      end
+
+      # Exposes the block's parameter list so KeywordFilter can filter
+      # kwargs to match the block's declared keywords.
+      # @return [Array<Array>] parameter list from Proc#parameters
+      def call_parameters
+        @block.parameters
+      end
+
+      private
+
+      # Legacy positional wrapping: arity 1/-1 gets :input extracted.
+      # Keyword and zero-arity blocks are stored raw; KeywordFilter handles filtering at call time.
+      # @param block [Proc]
+      # @return [Proc]
+      def wrap_block(block)
+        params = block.parameters
+        if Internal::Callable::KeywordFilter.has_any_keywords?(params) || block.arity == 0
+          block
+        elsif block.arity == 1 || block.arity == -1
+          Log.warn_once(:task_positional, "Task with positional param (input) is deprecated. Use keyword args: ->(input:) { ... } instead.")
+          ->(**kw) { block.call(kw[:input]) }
+        else
+          raise ArgumentError, "Task must accept keyword args or 1 positional param (got arity #{block.arity})"
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Braintrust
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: braintrust
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Braintrust
@@ -193,6 +193,7 @@ files:
 - lib/braintrust/api/datasets.rb
 - lib/braintrust/api/functions.rb
 - lib/braintrust/api/internal/auth.rb
+- lib/braintrust/api/internal/btql.rb
 - lib/braintrust/api/internal/experiments.rb
 - lib/braintrust/api/internal/projects.rb
 - lib/braintrust/config.rb
@@ -234,6 +235,7 @@ files:
 - lib/braintrust/eval.rb
 - lib/braintrust/eval/case.rb
 - lib/braintrust/eval/cases.rb
+- lib/braintrust/eval/context.rb
 - lib/braintrust/eval/evaluator.rb
 - lib/braintrust/eval/formatter.rb
 - lib/braintrust/eval/functions.rb
@@ -241,6 +243,9 @@ files:
 - lib/braintrust/eval/runner.rb
 - lib/braintrust/eval/scorer.rb
 - lib/braintrust/eval/summary.rb
+- lib/braintrust/eval/trace.rb
+- lib/braintrust/functions.rb
+- lib/braintrust/internal/callable.rb
 - lib/braintrust/internal/encoding.rb
 - lib/braintrust/internal/env.rb
 - lib/braintrust/internal/http.rb
@@ -250,6 +255,7 @@ files:
 - lib/braintrust/internal/time.rb
 - lib/braintrust/logger.rb
 - lib/braintrust/prompt.rb
+- lib/braintrust/scorer.rb
 - lib/braintrust/server.rb
 - lib/braintrust/server/auth/clerk_token.rb
 - lib/braintrust/server/auth/no_auth.rb
@@ -264,6 +270,7 @@ files:
 - lib/braintrust/server/sse.rb
 - lib/braintrust/setup.rb
 - lib/braintrust/state.rb
+- lib/braintrust/task.rb
 - lib/braintrust/trace.rb
 - lib/braintrust/trace/attachment.rb
 - lib/braintrust/trace/span_exporter.rb