braintrust 0.0.3 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 127f10c355ef8d5b0968dcb3197d9612a68455087ad704fa17e5dcb41512ad6d
-  data.tar.gz: 0e1d31073d9d71f43a74f7d4b37cea8644b119afc282501ee4b311c6cad059ad
+  metadata.gz: 6321acf7b780922ed97ea3cc57dde47a52947a10650a082dcfd9af780056d99a
+  data.tar.gz: 67c181e53537829931de704c7503cc056646652f9c1a61d914bc1ee0b7af69a2
 SHA512:
-  metadata.gz: 654fae04c4cf51fa32b27864b92ac832e3e37472bfaabe20871aa1899ba027ae4bff6e0a054f833fcb7afe3ef0d3870479ecb824f4c7af8180ca8ea65b21a41c
-  data.tar.gz: c03683f9793b38477986ade0694f38178434b70c1eea7a1870c2b80a89ad45278fe54f5c7f880eec22719ca0698fab0ad8de5efba5123ee1692c18b0a258d94c
+  metadata.gz: bb8546fdbf0a448016a1d31ceb8729a40be59e0d8d081ef275f763a11dbb2f5df0134ec52fc3b1c15c41d9dcdf42fbbe6becaf00ab7ac882c8f2f7e173a9a61f
+  data.tar.gz: 41e6d13504302a3b3ec26697cb50ce4736040d910c8e293d37088311922daa77274fc4214c86543ca21209bf56e82b2f6f00d5fbc2d7d0d0baad6f1e77cc48ff

data/README.md

@@ -1,7 +1,7 @@
 # Braintrust Ruby SDK
 
 [![Gem Version](https://img.shields.io/gem/v/braintrust.svg)](https://rubygems.org/gems/braintrust)
-[![Documentation](https://img.shields.io/badge/docs-rubydoc.info-blue.svg)](https://rubydoc.info/gems/braintrust)
+[![Documentation](https://img.shields.io/badge/docs-gemdocs.org-blue.svg)](https://gemdocs.org/gems/braintrust/)
 ![Beta](https://img.shields.io/badge/status-beta-yellow)
 
 ## Overview
@@ -171,6 +171,56 @@ puts "View trace at: #{Braintrust::Trace.permalink(root_span)}"
 OpenTelemetry.tracer_provider.shutdown
 ```
 
+### Attachments
+
+Attachments allow you to log binary data (images, PDFs, audio, etc.) as part of your traces. This is particularly useful for multimodal AI applications like vision models.
+
+```ruby
+require "braintrust"
+require "braintrust/trace/attachment"
+
+Braintrust.init
+
+tracer = OpenTelemetry.tracer_provider.tracer("vision-app")
+
+tracer.in_span("analyze-image") do |span|
+  # Create attachment from file
+  att = Braintrust::Trace::Attachment.from_file(
+    Braintrust::Trace::Attachment::IMAGE_PNG,
+    "./photo.png"
+  )
+
+  # Build message with attachment (OpenAI/Anthropic format)
+  messages = [
+    {
+      role: "user",
+      content: [
+        {type: "text", text: "What's in this image?"},
+        att.to_h  # Converts to {"type" => "base64_attachment", "content" => "data:..."}
+      ]
+    }
+  ]
+
+  # Log to trace
+  span.set_attribute("braintrust.input_json", JSON.generate(messages))
+end
+
+OpenTelemetry.tracer_provider.shutdown
+```
+
+You can create attachments from bytes, files, or URLs:
+
+```ruby
+# From bytes
+att = Braintrust::Trace::Attachment.from_bytes("image/jpeg", image_data)
+
+# From file
+att = Braintrust::Trace::Attachment.from_file("application/pdf", "./doc.pdf")
+
+# From URL
+att = Braintrust::Trace::Attachment.from_url("https://example.com/image.png")
+```
+
 ## Features
 
 - **Evaluations**: Run systematic evaluations of your AI systems with custom scoring functions
@@ -187,13 +237,14 @@ Check out the [`examples/`](./examples/) directory for complete working examples
 - [trace.rb](./examples/trace.rb) - Manual span creation and tracing
 - [openai.rb](./examples/openai.rb) - Automatically trace OpenAI API calls
 - [anthropic.rb](./examples/anthropic.rb) - Automatically trace Anthropic API calls
+- [trace/trace_attachments.rb](./examples/trace/trace_attachments.rb) - Log attachments (images, PDFs) in traces
 - [eval/dataset.rb](./examples/eval/dataset.rb) - Run evaluations using datasets stored in Braintrust
 - [eval/remote_functions.rb](./examples/eval/remote_functions.rb) - Use remote scoring functions
 
 ## Documentation
 
 - [Braintrust Documentation](https://www.braintrust.dev/docs)
-- [API Documentation](https://rubydoc.info/gems/braintrust)
+- [API Documentation](https://gemdocs.org/gems/braintrust/)
 
 ## Contributing
 

data/lib/braintrust/config.rb

@@ -4,14 +4,18 @@ module Braintrust
   # Configuration object that reads from environment variables
   # and allows overriding with explicit options
   class Config
-    attr_reader :api_key, :org_name, :default_project, :app_url, :api_url
+    attr_reader :api_key, :org_name, :default_project, :app_url, :api_url,
+      :filter_ai_spans, :span_filter_funcs
 
-    def initialize(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil)
+    def initialize(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil,
+      filter_ai_spans: nil, span_filter_funcs: nil)
       @api_key = api_key
       @org_name = org_name
       @default_project = default_project
       @app_url = app_url
       @api_url = api_url
+      @filter_ai_spans = filter_ai_spans
+      @span_filter_funcs = span_filter_funcs || []
     end
 
     # Create a Config from environment variables, with option overrides
@@ -21,14 +25,27 @@ module Braintrust
     # @param default_project [String, nil] Default project (overrides BRAINTRUST_DEFAULT_PROJECT env var)
     # @param app_url [String, nil] App URL (overrides BRAINTRUST_APP_URL env var)
     # @param api_url [String, nil] API URL (overrides BRAINTRUST_API_URL env var)
+    # @param filter_ai_spans [Boolean, nil] Enable AI span filtering (overrides BRAINTRUST_OTEL_FILTER_AI_SPANS env var)
+    # @param span_filter_funcs [Array<Proc>, nil] Custom span filter functions
     # @return [Config] the created config
-    def self.from_env(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil)
+    def self.from_env(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil,
+      filter_ai_spans: nil, span_filter_funcs: nil)
+      # Parse filter_ai_spans from ENV if not explicitly provided
+      env_filter_ai_spans = ENV["BRAINTRUST_OTEL_FILTER_AI_SPANS"]
+      filter_ai_spans_value = if filter_ai_spans.nil?
+        env_filter_ai_spans&.downcase == "true"
+      else
+        filter_ai_spans
+      end
+
       new(
         api_key: api_key || ENV["BRAINTRUST_API_KEY"],
         org_name: org_name || ENV["BRAINTRUST_ORG_NAME"],
         default_project: default_project || ENV["BRAINTRUST_DEFAULT_PROJECT"],
         app_url: app_url || ENV["BRAINTRUST_APP_URL"] || "https://www.braintrust.dev",
-        api_url: api_url || ENV["BRAINTRUST_API_URL"] || "https://api.braintrust.dev"
+        api_url: api_url || ENV["BRAINTRUST_API_URL"] || "https://api.braintrust.dev",
+        filter_ai_spans: filter_ai_spans_value,
+        span_filter_funcs: span_filter_funcs
       )
     end
   end

data/lib/braintrust/eval.rb

@@ -9,6 +9,170 @@ require "opentelemetry/sdk"
 require "json"
 
 module Braintrust
+  # Evaluation framework for testing AI systems with custom test cases and scoring functions.
+  #
+  # The Eval module provides tools for running systematic evaluations of your AI systems. An
+  # evaluation consists of:
+  # - **Cases**: Test inputs with optional expected outputs
+  # - **Task**: The code/model being evaluated
+  # - **Scorers**: Functions that judge the quality of outputs
+  #
+  # @example Basic evaluation with inline cases
+  #   require "braintrust"
+  #
+  #   Braintrust.init
+  #
+  #   # Define a simple task (the code being evaluated)
+  #   task = ->(input) { input.include?("a") ? "fruit" : "vegetable" }
+  #
+  #   # Run evaluation with inline cases
+  #   Braintrust::Eval.run(
+  #     project: "my-project",
+  #     experiment: "food-classifier",
+  #     cases: [
+  #       {input: "apple", expected: "fruit"},
+  #       {input: "carrot", expected: "vegetable"},
+  #       {input: "banana", expected: "fruit"}
+  #     ],
+  #     task: task,
+  #     scorers: [
+  #       # Named scorer with Eval.scorer
+  #       Braintrust::Eval.scorer("exact_match") do |input, expected, output|
+  #         output == expected ? 1.0 : 0.0
+  #       end
+  #     ]
+  #   )
+  #
+  # @example Different ways to define scorers (recommended patterns)
+  #   # Method reference (auto-uses method name as scorer name)
+  #   def exact_match(input, expected, output)
+  #     output == expected ? 1.0 : 0.0
+  #   end
+  #
+  #   # Named scorer with Eval.scorer
+  #   case_insensitive = Braintrust::Eval.scorer("case_insensitive") do |input, expected, output|
+  #     output.downcase == expected.downcase ? 1.0 : 0.0
+  #   end
+  #
+  #   # Callable class with name method
+  #   class FuzzyMatch
+  #     def name
+  #       "fuzzy_match"
+  #     end
+  #
+  #     def call(input, expected, output, metadata = {})
+  #       threshold = metadata[:threshold] || 0.8
+  #       # scoring logic here
+  #       1.0
+  #     end
+  #   end
+  #
+  #   # Anonymous lambda that returns named score object
+  #   multi_score = ->(input, expected, output) {
+  #     [
+  #       {name: "exact_match", score: output == expected ? 1.0 : 0.0},
+  #       {name: "length_match", score: output.length == expected.length ? 1.0 : 0.0}
+  #     ]
+  #   }
+  #
+  #   # All can be used together
+  #   Braintrust::Eval.run(
+  #     project: "my-project",
+  #     experiment: "scorer-examples",
+  #     cases: [{input: "test", expected: "test"}],
+  #     task: ->(input) { input },
+  #     scorers: [method(:exact_match), case_insensitive, FuzzyMatch.new, multi_score]
+  #   )
+  #
+  # @example Different ways to define tasks
+  #   # Lambda
+  #   task_lambda = ->(input) { "result" }
+  #
+  #   # Proc
+  #   task_proc = proc { |input| "result" }
+  #
+  #   # Method reference
+  #   def my_task(input)
+  #     "result"
+  #   end
+  #   task_method = method(:my_task)
+  #
+  #   # Callable class
+  #   class MyTask
+  #     def call(input)
+  #       "result"
+  #     end
+  #   end
+  #   task_class = MyTask.new
+  #
+  #   # All of these can be used as the task parameter
+  #   Braintrust::Eval.run(
+  #     project: "my-project",
+  #     experiment: "task-examples",
+  #     cases: [{input: "test"}],
+  #     task: task_lambda, # or task_proc, task_method, task_class
+  #     scorers: [
+  #       Braintrust::Eval.scorer("my_scorer") { |input, expected, output| 1.0 }
+  #     ]
+  #   )
+  #
+  # @example Using datasets instead of inline cases
+  #   # Fetch cases from a dataset stored in Braintrust
+  #   Braintrust::Eval.run(
+  #     project: "my-project",
+  #     experiment: "with-dataset",
+  #     dataset: "my-dataset-name", # fetches from same project
+  #     task: ->(input) { "result" },
+  #     scorers: [
+  #       Braintrust::Eval.scorer("my_scorer") { |input, expected, output| 1.0 }
+  #     ]
+  #   )
+  #
+  #   # Or with more options
+  #   Braintrust::Eval.run(
+  #     project: "my-project",
+  #     experiment: "with-dataset-options",
+  #     dataset: {
+  #       name: "my-dataset",
+  #       project: "other-project",
+  #       version: "1.0",
+  #       limit: 100
+  #     },
+  #     task: ->(input) { "result" },
+  #     scorers: [
+  #       Braintrust::Eval.scorer("my_scorer") { |input, expected, output| 1.0 }
+  #     ]
+  #   )
+  #
+  # @example Using metadata and tags
+  #   Braintrust::Eval.run(
+  #     project: "my-project",
+  #     experiment: "with-metadata",
+  #     cases: [
+  #       {
+  #         input: "apple",
+  #         expected: "fruit",
+  #         tags: ["tropical", "sweet"],
+  #         metadata: {threshold: 0.9, category: "produce"}
+  #       }
+  #     ],
+  #     task: ->(input) { "fruit" },
+  #     scorers: [
+  #       # Scorer can access case metadata
+  #       Braintrust::Eval.scorer("threshold_match") do |input, expected, output, metadata|
+  #         threshold = metadata[:threshold] || 0.5
+  #         # scoring logic using threshold
+  #         1.0
+  #       end
+  #     ],
+  #     # Experiment-level tags and metadata
+  #     tags: ["v1", "production"],
+  #     metadata: {
+  #       model: "gpt-4",
+  #       temperature: 0.7,
+  #       version: "1.0.0"
+  #     }
+  #   )
   module Eval
     class << self
       # Create a scorer with a name and callable

data/lib/braintrust/state.rb

@@ -6,7 +6,7 @@ module Braintrust
   # State object that holds Braintrust configuration
   # Thread-safe global state management
   class State
-    attr_reader :api_key, :org_name, :org_id, :default_project, :app_url, :api_url, :proxy_url, :logged_in
+    attr_reader :api_key, :org_name, :org_id, :default_project, :app_url, :api_url, :proxy_url, :logged_in, :config
 
     @mutex = Mutex.new
     @global_state = nil
@@ -20,15 +20,20 @@ module Braintrust
     # @param blocking_login [Boolean] whether to block and login synchronously (default: false)
     # @param enable_tracing [Boolean] whether to enable OpenTelemetry tracing (default: true)
     # @param tracer_provider [TracerProvider, nil] Optional tracer provider to use
+    # @param filter_ai_spans [Boolean, nil] Enable AI span filtering
+    # @param span_filter_funcs [Array<Proc>, nil] Custom span filter functions
+    # @param exporter [Exporter, nil] Optional exporter override (for testing)
     # @return [State] the created state
-    def self.from_env(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil, blocking_login: false, enable_tracing: true, tracer_provider: nil)
+    def self.from_env(api_key: nil, org_name: nil, default_project: nil, app_url: nil, api_url: nil, blocking_login: false, enable_tracing: true, tracer_provider: nil, filter_ai_spans: nil, span_filter_funcs: nil, exporter: nil)
       require_relative "config"
       config = Config.from_env(
         api_key: api_key,
         org_name: org_name,
         default_project: default_project,
         app_url: app_url,
-        api_url: api_url
+        api_url: api_url,
+        filter_ai_spans: filter_ai_spans,
+        span_filter_funcs: span_filter_funcs
       )
       new(
         api_key: config.api_key,
@@ -38,11 +43,13 @@ module Braintrust
         api_url: config.api_url,
         blocking_login: blocking_login,
         enable_tracing: enable_tracing,
-        tracer_provider: tracer_provider
+        tracer_provider: tracer_provider,
+        config: config,
+        exporter: exporter
       )
     end
 
-    def initialize(api_key: nil, org_name: nil, org_id: nil, default_project: nil, app_url: nil, api_url: nil, proxy_url: nil, blocking_login: false, enable_tracing: true, tracer_provider: nil)
+    def initialize(api_key: nil, org_name: nil, org_id: nil, default_project: nil, app_url: nil, api_url: nil, proxy_url: nil, blocking_login: false, enable_tracing: true, tracer_provider: nil, config: nil, exporter: nil)
       # Instance-level mutex for thread-safe login
       @login_mutex = Mutex.new
       raise ArgumentError, "api_key is required" if api_key.nil? || api_key.empty?
@@ -55,6 +62,7 @@ module Braintrust
       @api_url = api_url
       @proxy_url = proxy_url
       @logged_in = false
+      @config = config
 
       # Perform login after state setup
       if blocking_login
@@ -66,7 +74,7 @@ module Braintrust
       # Setup tracing if requested
       if enable_tracing
         require_relative "trace"
-        Trace.setup(self, tracer_provider)
+        Trace.setup(self, tracer_provider, exporter: exporter)
       end
     end
 

data/lib/braintrust/trace/attachment.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require "base64"
+require "net/http"
+require "uri"
+
+module Braintrust
+  module Trace
+    # Attachment represents binary data (images, audio, PDFs, etc.) that can be logged
+    # as part of traces in Braintrust. Attachments are stored securely and can be viewed
+    # in the Braintrust UI.
+    #
+    # Attachments are particularly useful for multimodal AI applications, such as vision
+    # models that process images.
+    #
+    # @example Create attachment from file
+    #   att = Braintrust::Trace::Attachment.from_file("image/png", "./photo.png")
+    #   data_url = att.to_data_url
+    #   # => "data:image/png;base64,iVBORw0KGgo..."
+    #
+    # @example Create attachment from bytes
+    #   att = Braintrust::Trace::Attachment.from_bytes("image/jpeg", image_bytes)
+    #   message = att.to_message
+    #   # => {"type" => "base64_attachment", "content" => "data:image/jpeg;base64,..."}
+    #
+    # @example Use in a trace span
+    #   att = Braintrust::Trace::Attachment.from_file("image/png", "./photo.png")
+    #   messages = [
+    #     {
+    #       role: "user",
+    #       content: [
+    #         {type: "text", text: "What's in this image?"},
+    #         att.to_h  # Converts to {"type" => "base64_attachment", "content" => "..."}
+    #       ]
+    #     }
+    #   ]
+    #   span.set_attribute("braintrust.input_json", JSON.generate(messages))
+    class Attachment
+      # Common MIME type constants for convenience
+      IMAGE_PNG = "image/png"
+      IMAGE_JPEG = "image/jpeg"
+      IMAGE_JPG = "image/jpg"
+      IMAGE_GIF = "image/gif"
+      IMAGE_WEBP = "image/webp"
+      TEXT_PLAIN = "text/plain"
+      APPLICATION_PDF = "application/pdf"
+
+      # @!visibility private
+      def initialize(content_type, data)
+        @content_type = content_type
+        @data = data
+      end
+
+      # Creates an attachment from raw bytes.
+      #
+      # @param content_type [String] MIME type of the data (e.g., "image/png")
+      # @param data [String] Binary data as a string
+      # @return [Attachment] New attachment instance
+      #
+      # @example
+      #   image_data = File.binread("photo.png")
+      #   att = Braintrust::Trace::Attachment.from_bytes("image/png", image_data)
+      def self.from_bytes(content_type, data)
+        new(content_type, data)
+      end
+
+      # Creates an attachment by reading from a file.
+      #
+      # @param content_type [String] MIME type of the file (e.g., "image/png")
+      # @param path [String] Path to the file to read
+      # @return [Attachment] New attachment instance
+      # @raise [Errno::ENOENT] If the file does not exist
+      #
+      # @example
+      #   att = Braintrust::Trace::Attachment.from_file("image/png", "./photo.png")
+      def self.from_file(content_type, path)
+        data = File.binread(path)
+        new(content_type, data)
+      end
+
+      # Creates an attachment by fetching data from a URL.
+      #
+      # The content type is inferred from the Content-Type header in the HTTP response.
+      # If the header is not present, it falls back to "application/octet-stream".
+      #
+      # @param url [String] URL to fetch
+      # @return [Attachment] New attachment instance
+      # @raise [StandardError] If the HTTP request fails
+      #
+      # @example
+      #   att = Braintrust::Trace::Attachment.from_url("https://example.com/image.png")
+      def self.from_url(url)
+        uri = URI.parse(url)
+        response = Net::HTTP.get_response(uri)
+
+        unless response.is_a?(Net::HTTPSuccess)
+          raise StandardError, "Failed to fetch URL: #{response.code} #{response.message}"
+        end
+
+        content_type = response.content_type || "application/octet-stream"
+        new(content_type, response.body)
+      end
+
+      # Converts the attachment to a data URL format.
+      #
+      # @return [String] Data URL in the format "data:<content-type>;base64,<encoded-data>"
+      #
+      # @example
+      #   att = Braintrust::Trace::Attachment.from_bytes("image/png", image_data)
+      #   att.to_data_url
+      #   # => "data:image/png;base64,iVBORw0KGgo..."
+      def to_data_url
+        encoded = Base64.strict_encode64(@data)
+        "data:#{@content_type};base64,#{encoded}"
+      end
+
+      # Converts the attachment to a message format suitable for LLM APIs.
+      #
+      # @return [Hash] Message hash with "type" and "content" keys
+      #
+      # @example
+      #   att = Braintrust::Trace::Attachment.from_bytes("image/png", image_data)
+      #   att.to_message
+      #   # => {"type" => "base64_attachment", "content" => "data:image/png;base64,..."}
+      def to_message
+        {
+          "type" => "base64_attachment",
+          "content" => to_data_url
+        }
+      end
+
+      # Alias for {#to_message}. Converts the attachment to a hash representation.
+      #
+      # @return [Hash] Same as {#to_message}
+      alias_method :to_h, :to_message
+    end
+  end
+end