braintrust 0.4.0 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0f30760b63f57dfa236f8f8f74c60aabad6e693f86a57bf8699b028eb00e8639
-  data.tar.gz: fcae112dc4175b2248a853405587921f16eb2c67d2b8930e2a3877cc09b9e9d1
+  metadata.gz: 1afc7372056311aec7ccaaae11c5d7a9204cdfc826a34db1bd67126769d253e3
+  data.tar.gz: 6492ea96b8f616c05241983baaeb935cdc3d206ced063781c44981b6d0817e64
 SHA512:
-  metadata.gz: 16e96c1f75646d2b581cb7a5c1c50ca66de3e625c75da11c6a9fd263313adea9a936e1e30f4a9733e16e56d3120737731d47ab89d858aafd7543b75011cbc9de
-  data.tar.gz: b926449904f3dafe6803f76105ee8d85b134c777b827c93b877390e318c167e1f8212eab7f207e1fc9a8e8b77cd6a2d48c5c374f412fc4af0a0f614a6c4de94e
+  metadata.gz: e6471fc6f3b8beba508dfef0deebfa5f10b1044aaf44817cfb341a71396655ec034b9eca70aaf2e4efb0afa9007a5c284564b3d3471ba1dd18edfd336ea51cb8
+  data.tar.gz: 333ad63d354f902c5e4c5d25d2d67e5ed4f4843fcbfeb7c33a6f273de9dcd7b1c77202417816dd9b834881a3d3a833c68418e81c9a1365ec2e503da3ada027ac

data/README.md

@@ -1,3 +1,5 @@
+[![Braintrust](./braintrust-logo.svg)](https://www.braintrust.dev/)
+
 # Braintrust Ruby SDK
 
 [![Gem Version](https://img.shields.io/gem/v/braintrust.svg)](https://rubygems.org/gems/braintrust)

data/lib/braintrust/config.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "internal/api_key_resolver"
+require_relative "internal/env"
 
 module Braintrust
   # Configuration object that reads from environment variables

data/lib/braintrust/dataset.rb

@@ -166,14 +166,14 @@ module Braintrust
       record
     end
 
-    # Build origin JSON for tracing/linking
+    # Build origin pointer for tracing/linking
     # @param raw [Hash] Raw record from API
     # @param dataset_id [String] Dataset ID (fallback if not in record)
-    # @return [String, nil] JSON-serialized origin, or nil if record lacks required fields
+    # @return [Hash, nil] Origin pointer, or nil if record lacks required fields
     def build_origin(raw, dataset_id)
       return nil unless raw["id"] && raw["_xact_id"]
 
-      Internal::Origin.to_json(
+      Internal::Origin.build(
         object_type: "dataset",
         object_id: raw["dataset_id"] || dataset_id,
         id: raw["id"],

data/lib/braintrust/eval/runner.rb

@@ -91,11 +91,11 @@ module Braintrust
           # Set attributes known before task execution
           eval_span.set_attribute("braintrust.parent", eval_context.parent_span_attr) if eval_context.parent_span_attr
           set_json_attr(eval_span, "braintrust.span_attributes", build_span_attributes("eval"))
-          set_json_attr(eval_span, "braintrust.input_json", {input: kase.input})
-          set_json_attr(eval_span, "braintrust.expected", kase.expected) if kase.expected
+          set_json_attr(eval_span, "braintrust.input_json", kase.input)
+          set_json_attr(eval_span, "braintrust.expected_json", kase.expected) unless kase.expected.nil?
           set_json_attr(eval_span, "braintrust.metadata", kase.metadata) if kase.metadata
           eval_span.set_attribute("braintrust.tags", kase.tags) if kase.tags
-          eval_span.set_attribute("braintrust.origin", kase.origin) if kase.origin
+          set_json_attr(eval_span, "braintrust.origin", kase.origin) if kase.origin
 
           # Run task
           begin
@@ -103,7 +103,6 @@ module Braintrust
           rescue => e
             # Error already recorded on task span, set eval span status
             eval_span.status = OpenTelemetry::Trace::Status.error(e.message)
-            set_json_attr(eval_span, "braintrust.output_json", {output: nil})
             errors << "Task failed for input '#{kase.input}': #{e.message}"
             report_progress(eval_span, kase, error: e.message)
             next
@@ -134,7 +133,7 @@ module Braintrust
           end
 
           # Set output after task completes
-          set_json_attr(eval_span, "braintrust.output_json", {output: kase.output})
+          set_json_attr(eval_span, "braintrust.output_json", kase.output)
 
           report_progress(eval_span, kase, data: kase.output)
         end
@@ -151,6 +150,7 @@ module Braintrust
           task_span.set_attribute("braintrust.parent", eval_context.parent_span_attr) if eval_context.parent_span_attr
           set_json_attr(task_span, "braintrust.span_attributes", build_span_attributes("task"))
           set_json_attr(task_span, "braintrust.input_json", kase.input)
+          set_json_attr(task_span, "braintrust.expected_json", kase.expected) unless kase.expected.nil?
 
           begin
             output = eval_context.task.call(
@@ -275,9 +275,7 @@ module Braintrust
       def report_progress(eval_span, kase, **fields)
         return unless eval_context.on_progress
         progress = {"id" => eval_span.context.hex_span_id}.merge(fields.transform_keys(&:to_s))
-        if kase.origin
-          progress["origin"] = kase.origin.is_a?(String) ? JSON.parse(kase.origin) : kase.origin
-        end
+        progress["origin"] = kase.origin if kase.origin
         eval_context.on_progress.call(progress)
       rescue => e
         Braintrust.logger.warn("on_progress callback error: #{e.message}")

data/lib/braintrust/eval.rb

@@ -336,7 +336,7 @@ module Braintrust
         # Use pinned version if available, otherwise compute from max(_xact_id)
         version = dataset_obj.version
         version ||= cases
-          .filter_map { |c| c[:origin] && JSON.parse(c[:origin])["_xact_id"] }
+          .filter_map { |c| c[:origin]&.dig("_xact_id") }
           .max
 
         {cases: cases, dataset_id: dataset_obj.id, dataset_version: version}

data/lib/braintrust/internal/env.rb

@@ -5,6 +5,8 @@ module Braintrust
     # Environment variable utilities.
     module Env
       ENV_AUTO_INSTRUMENT = "BRAINTRUST_AUTO_INSTRUMENT"
+      ENV_ENVIRONMENT_NAME = "BRAINTRUST_ENVIRONMENT_NAME"
+      ENV_ENVIRONMENT_TYPE = "BRAINTRUST_ENVIRONMENT_TYPE"
       ENV_INSTRUMENT_EXCEPT = "BRAINTRUST_INSTRUMENT_EXCEPT"
       ENV_INSTRUMENT_ONLY = "BRAINTRUST_INSTRUMENT_ONLY"
       ENV_FLUSH_ON_EXIT = "BRAINTRUST_FLUSH_ON_EXIT"
@@ -26,6 +28,36 @@ module Braintrust
         parse_list(ENV_INSTRUMENT_ONLY)
       end
 
+      def self.detect_environment
+        env_type = env_value(ENV_ENVIRONMENT_TYPE)
+        env_name = env_value(ENV_ENVIRONMENT_NAME)
+        if present?(env_type) || present?(env_name)
+          return {type: env_type, name: env_name}.compact
+        end
+
+        {
+          "GITHUB_ACTIONS" => "github_actions",
+          "GITLAB_CI" => "gitlab_ci",
+          "CIRCLECI" => "circleci",
+          "BUILDKITE" => "buildkite",
+          "JENKINS_URL" => "jenkins",
+          "JENKINS_HOME" => "jenkins",
+          "TF_BUILD" => "azure_pipelines",
+          "TEAMCITY_VERSION" => "teamcity",
+          "TRAVIS" => "travis",
+          "BITBUCKET_BUILD_NUMBER" => "bitbucket"
+        }.each do |key, name|
+          return {type: "ci", name: name} if present?(ENV[key])
+        end
+        return {type: "ci", name: "ci"} if present?(ENV["CI"])
+
+        server_name = detect_server_environment_name
+        return {type: "server", name: server_name} if server_name
+
+        deployment_mode_environment(ENV["RAILS_ENV"]) ||
+          deployment_mode_environment(ENV["RACK_ENV"])
+      end
+
       # Parse a comma-separated environment variable into an array of symbols.
       # @param key [String] The environment variable name
       # @return [Array<Symbol>, nil] Array of symbols, or nil if not set
@@ -34,6 +66,55 @@ module Braintrust
         return nil unless value
         value.split(",").map(&:strip).map(&:to_sym)
       end
+
+      def self.deployment_mode_environment(value)
+        return nil unless present?(value)
+
+        normalized = value.strip.downcase
+        return {type: "server", name: normalized} if ["production", "staging"].include?(normalized)
+        return {type: "local", name: normalized} if ["development", "local"].include?(normalized)
+
+        nil
+      end
+      private_class_method :deployment_mode_environment
+
+      def self.detect_server_environment_name
+        {"VERCEL" => "vercel", "NETLIFY" => "netlify"}.each do |key, name|
+          return name if present?(ENV[key])
+        end
+        return "ecs" if present?(ENV["ECS_CONTAINER_METADATA_URI"]) || present?(ENV["ECS_CONTAINER_METADATA_URI_V4"])
+
+        aws_execution_env = env_value("AWS_EXECUTION_ENV")
+        return "ecs" if aws_execution_env&.start_with?("AWS_ECS_")
+        return "aws_lambda" if aws_execution_env&.start_with?("AWS_Lambda_")
+        return "aws_lambda" if present?(ENV["AWS_LAMBDA_FUNCTION_NAME"])
+
+        {
+          "K_SERVICE" => "cloud_run",
+          "FUNCTION_TARGET" => "gcp_functions",
+          "KUBERNETES_SERVICE_HOST" => "kubernetes",
+          "DYNO" => "heroku",
+          "FLY_APP_NAME" => "fly",
+          "RAILWAY_ENVIRONMENT" => "railway",
+          "RENDER_SERVICE_NAME" => "render"
+        }.each do |key, name|
+          return name if present?(ENV[key])
+        end
+
+        nil
+      end
+      private_class_method :detect_server_environment_name
+
+      def self.env_value(key)
+        value = ENV[key]
+        value&.strip unless value.nil? || value.strip.empty?
+      end
+      private_class_method :env_value
+
+      def self.present?(value)
+        !value.nil? && !value.strip.empty?
+      end
+      private_class_method :present?
     end
   end
 end

data/lib/braintrust/internal/origin.rb

@@ -1,27 +1,27 @@
 # frozen_string_literal: true
 
-require "json"
-
 module Braintrust
   module Internal
-    # Origin provides serialization for source object pointers in Braintrust.
-    # Used internally to link spans back to their source records (e.g., dataset rows).
+    # Origin builds source object pointers, which link spans back to the record
+    # they came from (e.g., a dataset row). Pointers stay Hashes internally and
+    # are serialized once at the boundary that needs them, so that inbound
+    # pointers from the wire and ones we build here have the same shape.
     module Origin
-      # Serialize an origin pointer to JSON
+      # Build an origin pointer
       # @param object_type [String] Type of source object (e.g., "dataset", "playground_logs")
       # @param object_id [String] ID of the source object
       # @param id [String] ID of the specific record within the source
       # @param xact_id [String] Transaction ID
       # @param created [String, nil] Creation timestamp
-      # @return [String] JSON-serialized origin
-      def self.to_json(object_type:, object_id:, id:, xact_id:, created:)
-        JSON.dump({
-          object_type: object_type,
-          object_id: object_id,
-          id: id,
-          _xact_id: xact_id,
-          created: created
-        })
+      # @return [Hash] Origin pointer with string keys, matching the wire format
+      def self.build(object_type:, object_id:, id:, xact_id:, created:)
+        {
+          "object_type" => object_type,
+          "object_id" => object_id,
+          "id" => id,
+          "_xact_id" => xact_id,
+          "created" => created
+        }
       end
     end
   end

data/lib/braintrust/server/services/eval_service.rb

@@ -2,12 +2,19 @@
 
 require "json"
 
+require_relative "../../eval/case"
+
 module Braintrust
   module Server
     module Services
       # Framework-agnostic service for running evaluations and streaming SSE results.
       # Must be long-lived (not per-request) to preserve the @state_cache across requests.
       class Eval
+        # Row fields this SDK understands, derived from the Case contract so that
+        # adding a field there carries it through here. Anything else on an
+        # inbound row is a field this version has no code for, and is ignored.
+        CASE_FIELDS = Braintrust::Eval::Case.members.map(&:to_s).freeze
+
         def initialize(evaluators)
           @evaluators = evaluators
           @state_mutex = Mutex.new
@@ -177,9 +184,10 @@ module Braintrust
         # Returns [cases, dataset] where exactly one is non-nil.
         def resolve_data_source(data)
           if data.key?("data")
-            cases = data["data"].map do |d|
-              {input: d["input"], expected: d["expected"]}
-            end
+            # Rows arrive inline from the Playground carrying tags, metadata and
+            # an origin pointer back to the row they came from. Carry them all:
+            # the Playground matches streamed results to its grid by origin.
+            cases = data["data"].map { |row| row.slice(*CASE_FIELDS).transform_keys(&:to_sym) }
             [cases, nil]
           elsif data.key?("dataset_id")
             [nil, Braintrust::Dataset::ID.new(id: data["dataset_id"])]

data/lib/braintrust/trace/span_exporter.rb

@@ -2,16 +2,20 @@
 
 require "opentelemetry/exporter/otlp"
 require_relative "../state"
+require_relative "span_origin"
 
 module Braintrust
   module Trace
-    # Custom OTLP exporter that groups spans by braintrust.parent attribute
-    # and sets the x-bt-parent HTTP header per group. This is required for
-    # the Braintrust OTLP backend to route spans to the correct experiment/project.
+    # Custom OTLP exporter for the Braintrust backend. On export it:
+    # - stamps span origin provenance onto each SpanData (via the prepended SpanOrigin behavior)
+    # - groups spans by braintrust.parent and sets the x-bt-parent header per group,
+    #   so the backend routes them to the correct experiment/project
     #
     # Thread safety: BatchSpanProcessor serializes export() calls via its
     # @export_mutex, so @headers mutation here is safe.
     class SpanExporter < OpenTelemetry::Exporter::OTLP::Exporter
+      prepend SpanOrigin
+
       PARENT_ATTR_KEY = SpanProcessor::PARENT_ATTR_KEY
       PARENT_HEADER = "x-bt-parent"
 

data/lib/braintrust/trace/span_filter.rb

@@ -16,7 +16,8 @@ module Braintrust
       SYSTEM_ATTRIBUTES = [
         "braintrust.parent",
         "braintrust.org",
-        "braintrust.app_url"
+        "braintrust.app_url",
+        "braintrust.context_json"
       ].freeze
 
       # Prefixes that indicate an AI-related span

data/lib/braintrust/trace/span_origin.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "json"
+require_relative "../version"
+require_relative "../internal/env"
+
+module Braintrust
+  module Trace
+    # Span origin provenance decoration.
+    #
+    # This is a *behavior*, not a type. Prepend it onto any exporter whose
+    # +export(span_data, timeout:)+ it can +super+ into, and every exported
+    # SpanData gains a +braintrust.context_json+ attribute carrying span origin
+    # (SDK name/version, instrumentation scope, environment).
+    #
+    # Because it only ever touches the SpanData copies handed to *this*
+    # exporter, the enrichment is invisible to any other exporter sharing the
+    # same tracer provider - there is no global patch and nothing leaks onto a
+    # customer's other OTel traces.
+    module SpanOrigin
+      CONTEXT_JSON_ATTR_KEY = "braintrust.context_json"
+
+      # Exporter behavior: enrich each SpanData with span origin before export.
+      # @param span_data [Array<OpenTelemetry::SDK::Trace::SpanData>]
+      # @return [Integer] export result from the wrapped exporter
+      def export(span_data, timeout: nil)
+        # Environment is process-global and stable; read it once per batch
+        # rather than once per span. It is cheap (ENV reads only).
+        environment = Internal::Env.detect_environment
+        enriched = span_data.map { |sd| SpanOrigin.enrich(sd, environment: environment) }
+        super(enriched, timeout: timeout)
+      end
+
+      # Enrich a single SpanData with span origin provenance.
+      # Mutates the SpanData in place (replacing its frozen attributes hash with
+      # a new frozen hash - it never mutates the shared hash) and returns it.
+      # @param span_data [OpenTelemetry::SDK::Trace::SpanData]
+      # @param environment [Hash, nil] process environment ({type:, name:}) or nil
+      # @return [OpenTelemetry::SDK::Trace::SpanData]
+      def self.enrich(span_data, environment:)
+        attributes = span_data.attributes || {}
+        enriched_attributes = attributes_with_origin(
+          attributes,
+          instrumentation_name: instrumentation_name(span_data),
+          environment: environment
+        )
+
+        return span_data if enriched_attributes.equal?(attributes)
+
+        span_data.attributes = enriched_attributes.freeze
+        span_data.total_recorded_attributes = enriched_attributes.length
+        span_data
+      end
+
+      def self.attributes_with_origin(attributes, instrumentation_name:, environment:)
+        context = parse_context_json(attributes[CONTEXT_JSON_ATTR_KEY])
+        span_origin = context["span_origin"].is_a?(Hash) ? context["span_origin"] : {}
+
+        span_origin_changed = false
+        unless span_origin.key?("name")
+          span_origin["name"] = "braintrust.sdk.ruby"
+          span_origin_changed = true
+        end
+        unless span_origin.key?("version")
+          span_origin["version"] = Braintrust::VERSION
+          span_origin_changed = true
+        end
+        unless span_origin.key?("instrumentation")
+          span_origin["instrumentation"] = {"name" => instrumentation_name}
+          span_origin_changed = true
+        end
+        if environment && !span_origin.key?("environment")
+          span_origin["environment"] = environment
+          span_origin_changed = true
+        end
+
+        context_changed = context["span_origin"] != span_origin || span_origin_changed
+        return attributes unless context_changed
+
+        context["span_origin"] = span_origin
+        attributes.merge(CONTEXT_JSON_ATTR_KEY => JSON.generate(context))
+      end
+
+      def self.parse_context_json(raw)
+        return {} unless raw.is_a?(String) && !raw.strip.empty?
+
+        parsed = JSON.parse(raw)
+        parsed.is_a?(Hash) ? parsed : {}
+      rescue JSON::ParserError
+        {}
+      end
+
+      def self.instrumentation_name(span)
+        if span.respond_to?(:instrumentation_scope) && span.instrumentation_scope&.respond_to?(:name)
+          return span.instrumentation_scope.name
+        end
+        if span.respond_to?(:instrumentation_library) && span.instrumentation_library&.respond_to?(:name)
+          return span.instrumentation_library.name
+        end
+
+        "braintrust-ruby"
+      end
+    end
+  end
+end

data/lib/braintrust/trace/span_processor.rb

@@ -83,9 +83,11 @@ module Braintrust
         # If no filters, keep everything
         return true if @filters.empty?
 
+        span_data = span.respond_to?(:to_span_data) ? span.to_span_data : span
+
         # Apply filters in order - first non-zero result wins
         @filters.each do |filter|
-          result = filter.call(span)
+          result = filter.call(span_data)
           return true if result > 0  # Keep span
           return false if result < 0 # Drop span
           # result == 0: no influence, continue to next filter

data/lib/braintrust/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Braintrust
-  VERSION = "0.4.0"
+  VERSION = "0.4.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: braintrust
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.2
 platform: ruby
 authors:
 - Braintrust
@@ -182,6 +182,7 @@ files:
 - lib/braintrust/trace/attachment.rb
 - lib/braintrust/trace/span_exporter.rb
 - lib/braintrust/trace/span_filter.rb
+- lib/braintrust/trace/span_origin.rb
 - lib/braintrust/trace/span_processor.rb
 - lib/braintrust/vendor/mustache.rb
 - lib/braintrust/vendor/mustache/context.rb