braintrust 0.0.1.alpha.2

data/lib/braintrust/eval/functions.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require_relative "../api"
+require_relative "scorer"
+require "opentelemetry/sdk"
+require "json"
+
+module Braintrust
+  module Eval
+    # Functions provides remote function execution capabilities
+    # Allows calling prompts hosted on Braintrust servers as tasks or scorers
+    module Functions
+      class << self
+        # Create a task callable that invokes a remote function
+        # @param project [String] Project name
+        # @param slug [String] Function slug
+        # @param state [State, nil] Braintrust state (defaults to global)
+        # @param tracer_provider [TracerProvider, nil] OpenTelemetry tracer provider
+        # @return [Proc] Callable that accepts input and returns output
+        def task(project:, slug:, state: nil, tracer_provider: nil)
+          state ||= Braintrust.current_state
+          raise Error, "No state available" unless state
+
+          # Resolve function ID from project + slug
+          api = API.new(state: state)
+          function_metadata = resolve_function(api, project, slug)
+          function_id = function_metadata["id"]
+          function_name = function_metadata["name"] || slug
+
+          # Get tracer for creating spans
+          tracer_provider ||= OpenTelemetry.tracer_provider
+          tracer = tracer_provider.tracer("braintrust.functions")
+
+          # Return a lambda that invokes the remote function with tracing
+          lambda do |input|
+            # Create a span for the function invocation
+            tracer.in_span("function: #{slug}") do |span|
+              span.set_attribute("braintrust.span_attributes", JSON.dump({type: "function"}))
+              span.set_attribute("braintrust.input_json", JSON.dump(input))
+              span.set_attribute("braintrust.function.name", function_name)
+              span.set_attribute("braintrust.function.id", function_id)
+              span.set_attribute("braintrust.function.slug", slug)
+
+              begin
+                # Invoke the function via API
+                output = api.functions.invoke(id: function_id, input: input)
+                span.set_attribute("braintrust.output_json", JSON.dump(output))
+                output
+              rescue => e
+                # Record exception and set error status
+                span.record_exception(e)
+                span.status = OpenTelemetry::Trace::Status.error(e.message)
+                raise
+              end
+            end
+          end
+        end
+
+        # Create a scorer that invokes a remote function
+        # @param project [String] Project name
+        # @param slug [String] Function slug
+        # @param state [State, nil] Braintrust state (defaults to global)
+        # @param tracer_provider [TracerProvider, nil] OpenTelemetry tracer provider
+        # @return [Scorer] Scorer object that invokes remote function
+        def scorer(project:, slug:, state: nil, tracer_provider: nil)
+          state ||= Braintrust.current_state
+          raise Error, "No state available" unless state
+
+          # Resolve function ID from project + slug
+          api = API.new(state: state)
+          function_metadata = resolve_function(api, project, slug)
+          function_id = function_metadata["id"]
+          function_name = function_metadata["name"] || slug
+
+          # Get tracer for creating spans
+          tracer_provider ||= OpenTelemetry.tracer_provider
+          tracer = tracer_provider.tracer("braintrust.functions")
+
+          # Create a scorer that invokes the remote function
+          Scorer.new(slug) do |input, expected, output, metadata|
+            # Create a span for the function invocation
+            tracer.in_span("function: #{slug}") do |span|
+              scorer_input = {
+                input: input,
+                expected: expected,
+                output: output,
+                metadata: metadata
+              }
+
+              span.set_attribute("braintrust.span_attributes", JSON.dump({type: "function"}))
+              span.set_attribute("braintrust.input_json", JSON.dump(scorer_input))
+              span.set_attribute("braintrust.function.name", function_name)
+              span.set_attribute("braintrust.function.id", function_id)
+              span.set_attribute("braintrust.function.slug", slug)
+
+              begin
+                # Invoke the function via API
+                # The remote scorer receives all scorer arguments
+                result = api.functions.invoke(id: function_id, input: scorer_input)
+
+                # Parse result as float score
+                # The remote function should return a number
+                score = result.is_a?(Numeric) ? result.to_f : result.to_s.to_f
+
+                span.set_attribute("braintrust.output_json", JSON.dump(score))
+                score
+              rescue => e
+                # Record exception and set error status
+                span.record_exception(e)
+                span.status = OpenTelemetry::Trace::Status.error(e.message)
+                raise
+              end
+            end
+          end
+        end
+
+        private
+
+        # Resolve function ID from project name and slug
+        # @param api [API] API client
+        # @param project [String] Project name
+        # @param slug [String] Function slug
+        # @return [Hash] Function metadata
+        def resolve_function(api, project, slug)
+          result = api.functions.list(project_name: project, slug: slug)
+          functions = result["objects"]
+
+          if functions.nil? || functions.empty?
+            raise Error, "Function '#{slug}' not found in project '#{project}'"
+          end
+
+          functions.first
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/eval/result.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Eval
+    # Result represents the outcome of an evaluation run
+    # Contains experiment metadata, errors, and timing information
+    class Result
+      attr_reader :experiment_id, :experiment_name, :project_id,
+        :permalink, :errors, :duration
+
+      # Create a new result
+      # @param experiment_id [String] The experiment ID
+      # @param experiment_name [String] The experiment name
+      # @param project_id [String] The project ID
+      # @param permalink [String] Link to view the experiment in Braintrust UI
+      # @param errors [Array<String>] List of errors that occurred
+      # @param duration [Float] Duration in seconds
+      def initialize(experiment_id:, experiment_name:, project_id:,
+        permalink:, errors:, duration:)
+        @experiment_id = experiment_id
+        @experiment_name = experiment_name
+        @project_id = project_id
+        @permalink = permalink
+        @errors = errors
+        @duration = duration
+      end
+
+      # Check if the evaluation was successful (no errors)
+      # @return [Boolean]
+      def success?
+        errors.empty?
+      end
+
+      # Check if the evaluation failed (has errors)
+      # @return [Boolean]
+      def failed?
+        !success?
+      end
+
+      # Format the result as a human-readable string (Go SDK format)
+      # @return [String]
+      def to_s
+        [
+          "Experiment: #{experiment_name}",
+          "ID: #{experiment_id}",
+          "Link: #{permalink}",
+          "Duration: #{duration.round(2)}s",
+          "Errors: #{errors.length}"
+        ].join("\n")
+      end
+    end
+  end
+end

data/lib/braintrust/eval/scorer.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Eval
+    # Scorer wraps a scoring function that evaluates task output against expected values
+    # Scorers can accept 3 params (input, expected, output) or 4 params (input, expected, output, metadata)
+    # They can return a float, hash, or array of hashes
+    class Scorer
+      attr_reader :name
+
+      # Create a new scorer
+      # @param name_or_callable [String, Symbol, #call] Name or callable (if callable, name is auto-detected)
+      # @param callable [#call, nil] Callable if name was provided separately
+      # @param block [Proc, nil] Block if no callable provided
+      def initialize(name_or_callable = nil, callable = nil, &block)
+        # Determine name and callable from arguments
+        if name_or_callable.nil? && callable.nil? && block.nil?
+          raise ArgumentError, "Must provide callable or block"
+        end
+
+        # If first arg is a string/symbol, it's the name
+        if name_or_callable.is_a?(String) || name_or_callable.is_a?(Symbol)
+          @name = name_or_callable.to_s
+          @callable = callable || block
+          raise ArgumentError, "Must provide callable or block" unless @callable
+        else
+          # First arg is the callable, try to auto-detect name
+          @callable = name_or_callable || callable || block
+          @name = detect_name(@callable)
+        end
+
+        # Validate callable
+        unless @callable.respond_to?(:call)
+          raise ArgumentError, "Scorer must be callable (respond to :call)"
+        end
+
+        # Detect arity and wrap callable if needed
+        @wrapped_callable = wrap_callable(@callable)
+      end
+
+      # Call the scorer
+      # @param input [Object] The input to the task
+      # @param expected [Object] The expected output
+      # @param output [Object] The actual output from the task
+      # @param metadata [Hash] Optional metadata
+      # @return [Float, Hash, Array] Score value(s)
+      def call(input, expected, output, metadata = {})
+        @wrapped_callable.call(input, expected, output, metadata)
+      end
+
+      private
+
+      # Detect the name from a callable object
+      # @param callable [#call] The callable
+      # @return [String] The detected name
+      def detect_name(callable)
+        # Method objects have .name
+        if callable.is_a?(Method)
+          return callable.name.to_s
+        end
+
+        # Objects with .name method
+        if callable.respond_to?(:name)
+          return callable.name.to_s
+        end
+
+        # Fallback
+        "scorer"
+      end
+
+      # Wrap the callable to always accept 4 parameters
+      # @param callable [#call] The callable to wrap
+      # @return [Proc] Wrapped callable that accepts 4 params
+      def wrap_callable(callable)
+        arity = callable_arity(callable)
+
+        case arity
+        when 3
+          # Callable takes 3 params - wrap to ignore metadata
+          ->(input, expected, output, metadata) {
+            callable.call(input, expected, output)
+          }
+        when 4, -4, -1
+          # Callable takes 4 params (or variadic with 4+)
+          # -4 means optional 4th param
+          # -1 means variadic (*args)
+          callable
+        else
+          raise ArgumentError, "Scorer must accept 3 or 4 parameters (got arity #{arity})"
+        end
+      end
+
+      # Get the arity of a callable
+      # @param callable [#call] The callable
+      # @return [Integer] The arity
+      def callable_arity(callable)
+        if callable.respond_to?(:arity)
+          callable.arity
+        elsif callable.respond_to?(:method)
+          callable.method(:call).arity
+        else
+          # Assume 3 params if we can't detect
+          3
+        end
+      end
+    end
+  end
+end