braintrust 0.0.1.alpha.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2f2fd3e289b473dc2f40c2fe9df914c1b1b7ee2ac06db54656fa24eb06a3ff5a
+  data.tar.gz: 562aadfd39d224e5052685ed73c4f2a504db4df01fd26445bf4b0411a8987bb5
+SHA512:
+  metadata.gz: db90cc0be88cd3c5a88b59d0b87829c88cec76f15bb1213840efb239474a389c95be62ba41efcdb0b87e069845f71873ee7cfefc224e2ba1d4e5d7affbd536a3
+  data.tar.gz: 8ff1ace52cc31b7feae57c557e029f09f8fd811cc2163fb042a5b9e9965b6a9eef2f00e9c543d5c177ec52b04163b119bb5c68dfb57d73d72d7dc63b2974f213

data/README.md

@@ -0,0 +1,24 @@
+# Braintrust Ruby SDK
+
+Ruby SDK for [Braintrust](https://www.braintrust.dev) - AI evaluation and observability platform.
+
+## Status
+
+🚧 Under active development
+
+## Development
+
+```bash
+./scripts/install-deps.sh
+mise install
+cp .env.example .env
+rake test
+```
+
+## License
+
+Apache-2.0
+
+## Contributing
+
+See [CONTRIBUTING.md](CONTRIBUTING.md)

data/lib/braintrust/api/datasets.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "uri"
+require_relative "../logger"
+
+module Braintrust
+  class API
+    # Datasets API namespace
+    # Provides methods for creating, fetching, and querying datasets
+    class Datasets
+      def initialize(api)
+        @api = api
+        @state = api.state
+      end
+
+      # List datasets with optional filters
+      # GET /v1/dataset?project_name=X&dataset_name=Y&...
+      # @param project_name [String, nil] Filter by project name
+      # @param dataset_name [String, nil] Filter by dataset name
+      # @param project_id [String, nil] Filter by project ID
+      # @param limit [Integer, nil] Limit number of results
+      # @return [Hash] Response with "objects" array
+      def list(project_name: nil, dataset_name: nil, project_id: nil, limit: nil)
+        params = {}
+        params["project_name"] = project_name if project_name
+        params["dataset_name"] = dataset_name if dataset_name
+        params["project_id"] = project_id if project_id
+        params["limit"] = limit if limit
+
+        http_get("/v1/dataset", params)
+      end
+
+      # Fetch exactly one dataset by project + name (convenience method)
+      # @param project_name [String] Project name
+      # @param name [String] Dataset name
+      # @return [Hash] Dataset metadata
+      # @raise [Braintrust::Error] if dataset not found
+      def get(project_name:, name:)
+        result = list(project_name: project_name, dataset_name: name)
+        metadata = result["objects"]&.first
+        raise Error, "Dataset '#{name}' not found in project '#{project_name}'" unless metadata
+        metadata
+      end
+
+      # Fetch dataset metadata by ID
+      # GET /v1/dataset/{id}
+      # @param id [String] Dataset UUID
+      # @return [Hash] Dataset metadata
+      def get_by_id(id:)
+        http_get("/v1/dataset/#{id}")
+      end
+
+      # Create or register a dataset (idempotent)
+      # Uses app API /api/dataset/register which is idempotent - calling this method
+      # multiple times with the same name will return the existing dataset.
+      # @param project_name [String, nil] Project name
+      # @param project_id [String, nil] Project ID
+      # @param name [String] Dataset name
+      # @param description [String, nil] Optional description
+      # @param metadata [Hash, nil] Optional metadata
+      # @return [Hash] Response with "project", "dataset", and optional "found_existing" keys.
+      #   The "found_existing" field is true if the dataset already existed, false/nil if newly created.
+      def create(name:, project_name: nil, project_id: nil, description: nil, metadata: nil)
+        payload = {dataset_name: name, org_id: @state.org_id}
+        payload[:project_name] = project_name if project_name
+        payload[:project_id] = project_id if project_id
+        payload[:description] = description if description
+        payload[:metadata] = metadata if metadata
+
+        http_post_json_app("/api/dataset/register", payload)
+      end
+
+      # Insert events into a dataset
+      # POST /v1/dataset/{id}/insert
+      # @param id [String] Dataset UUID
+      # @param events [Array<Hash>] Array of event records
+      # @return [Hash] Insert response
+      def insert(id:, events:)
+        http_post_json("/v1/dataset/#{id}/insert", {events: events})
+      end
+
+      # Generate a permalink URL to view a dataset in the Braintrust UI
+      # @param id [String] Dataset UUID
+      # @return [String] Permalink URL
+      def permalink(id:)
+        "#{@state.app_url}/app/#{@state.org_name}/object?object_type=dataset&object_id=#{id}"
+      end
+
+      # Fetch records from dataset using BTQL
+      # POST /btql
+      # @param id [String] Dataset UUID
+      # @param limit [Integer] Max records per page (default: 1000)
+      # @param cursor [String, nil] Pagination cursor
+      # @param version [String, nil] Dataset version
+      # @return [Hash] Hash with :records array and :cursor string
+      def fetch(id:, limit: 1000, cursor: nil, version: nil)
+        query = {
+          from: {
+            op: "function",
+            name: {op: "ident", name: ["dataset"]},
+            args: [{op: "literal", value: id}]
+          },
+          select: [{op: "star"}],
+          limit: limit
+        }
+        query[:cursor] = cursor if cursor
+
+        payload = {query: query, fmt: "jsonl"}
+        payload[:version] = version if version
+
+        response = http_post_json_raw("/btql", payload)
+
+        # Parse JSONL response
+        records = response.body.lines
+          .map { |line| JSON.parse(line.strip) if line.strip.length > 0 }
+          .compact
+
+        # Extract pagination cursor from headers
+        next_cursor = response["x-bt-cursor"] || response["x-amz-meta-bt-cursor"]
+
+        {records: records, cursor: next_cursor}
+      end
+
+      private
+
+      # Core HTTP request method with logging
+      # @param method [Symbol] :get or :post
+      # @param path [String] API path
+      # @param params [Hash] Query params (for GET)
+      # @param payload [Hash, nil] JSON payload (for POST)
+      # @param base_url [String, nil] Override base URL (default: api_url)
+      # @param parse_json [Boolean] Whether to parse response as JSON (default: true)
+      # @return [Hash, Net::HTTPResponse] Parsed JSON or raw response
+      def http_request(method, path, params: {}, payload: nil, base_url: nil, parse_json: true)
+        # Build URI
+        base = base_url || @state.api_url
+        uri = URI("#{base}#{path}")
+        uri.query = URI.encode_www_form(params) unless params.empty?
+
+        # Create request
+        request = case method
+        when :get
+          Net::HTTP::Get.new(uri)
+        when :post
+          req = Net::HTTP::Post.new(uri)
+          req["Content-Type"] = "application/json"
+          req.body = JSON.dump(payload) if payload
+          req
+        else
+          raise ArgumentError, "Unsupported HTTP method: #{method}"
+        end
+
+        request["Authorization"] = "Bearer #{@state.api_key}"
+
+        # Execute request with timing
+        start_time = Time.now
+        Log.debug("[API] #{method.upcase} #{uri}")
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = (uri.scheme == "https")
+        response = http.request(request)
+
+        duration_ms = ((Time.now - start_time) * 1000).round(2)
+        Log.debug("[API] #{method.upcase} #{uri} -> #{response.code} (#{duration_ms}ms, #{response.body.bytesize} bytes)")
+
+        # Handle response
+        unless response.is_a?(Net::HTTPSuccess)
+          Log.debug("[API] Error response body: #{response.body}")
+          raise Error, "HTTP #{response.code} for #{method.upcase} #{uri}: #{response.body}"
+        end
+
+        parse_json ? JSON.parse(response.body) : response
+      end
+
+      # HTTP GET with query params - returns parsed JSON
+      def http_get(path, params = {})
+        http_request(:get, path, params: params)
+      end
+
+      # HTTP POST with JSON body - returns parsed JSON
+      def http_post_json(path, payload)
+        http_request(:post, path, payload: payload)
+      end
+
+      # HTTP POST to app URL (not API URL) - returns parsed JSON
+      def http_post_json_app(path, payload)
+        http_request(:post, path, payload: payload, base_url: @state.app_url)
+      end
+
+      # HTTP POST with JSON body - returns raw response (for header access)
+      def http_post_json_raw(path, payload)
+        http_request(:post, path, payload: payload, parse_json: false)
+      end
+    end
+  end
+end

data/lib/braintrust/api/functions.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "uri"
+require_relative "../logger"
+
+module Braintrust
+  class API
+    # Functions API namespace
+    # Provides methods for creating, invoking, and managing remote functions (prompts)
+    class Functions
+      def initialize(api)
+        @api = api
+        @state = api.state
+      end
+
+      # List functions with optional filters
+      # GET /v1/function?project_name=X&...
+      # @param project_name [String, nil] Filter by project name
+      # @param function_name [String, nil] Filter by function name
+      # @param slug [String, nil] Filter by slug
+      # @param limit [Integer, nil] Limit number of results
+      # @return [Hash] Response with "objects" array
+      def list(project_name: nil, function_name: nil, slug: nil, limit: nil)
+        params = {}
+        params["project_name"] = project_name if project_name
+        params["function_name"] = function_name if function_name
+        params["slug"] = slug if slug
+        params["limit"] = limit if limit
+
+        http_get("/v1/function", params)
+      end
+
+      # Create or register a function (idempotent)
+      # POST /v1/function
+      # This method is idempotent - if a function with the same slug already exists in the project,
+      # it will return the existing function unmodified. Unlike datasets, the response does not
+      # include a "found_existing" field.
+      # @param project_name [String] Project name
+      # @param slug [String] Function slug (URL-friendly identifier)
+      # @param function_data [Hash] Function configuration (usually {type: "prompt"})
+      # @param prompt_data [Hash, nil] Prompt configuration (prompt, options, etc.)
+      # @param name [String, nil] Optional display name (defaults to slug)
+      # @param description [String, nil] Optional description
+      # @return [Hash] Function metadata
+      def create(project_name:, slug:, function_data:, prompt_data: nil, name: nil, description: nil)
+        # Look up project ID
+        projects_result = http_get("/v1/project", {"project_name" => project_name})
+        project = projects_result["objects"]&.first
+        raise Error, "Project '#{project_name}' not found" unless project
+        project_id = project["id"]
+
+        payload = {
+          project_id: project_id,
+          slug: slug,
+          name: name || slug,  # Name is required, default to slug
+          function_data: function_data
+        }
+        payload[:prompt_data] = prompt_data if prompt_data
+        payload[:description] = description if description
+
+        http_post_json("/v1/function", payload)
+      end
+
+      # Invoke a function by ID with input
+      # POST /v1/function/{id}/invoke
+      # @param id [String] Function UUID
+      # @param input [Object] Input data to pass to the function
+      # @return [Object] The function output (String, Hash, Array, etc.) as returned by the HTTP API
+      def invoke(id:, input:)
+        payload = {input: input}
+        http_post_json("/v1/function/#{id}/invoke", payload)
+      end
+
+      # Delete a function by ID
+      # DELETE /v1/function/{id}
+      # @param id [String] Function UUID
+      # @return [Hash] Delete response
+      def delete(id:)
+        http_delete("/v1/function/#{id}")
+      end
+
+      private
+
+      # Core HTTP request method with logging
+      # @param method [Symbol] :get, :post, or :delete
+      # @param path [String] API path
+      # @param params [Hash] Query params (for GET)
+      # @param payload [Hash, nil] JSON payload (for POST)
+      # @param parse_json [Boolean] Whether to parse response as JSON (default: true)
+      # @return [Hash, Net::HTTPResponse] Parsed JSON or raw response
+      def http_request(method, path, params: {}, payload: nil, parse_json: true)
+        # Build URI
+        base = @state.api_url
+        uri = URI("#{base}#{path}")
+        uri.query = URI.encode_www_form(params) unless params.empty?
+
+        # Create request
+        request = case method
+        when :get
+          Net::HTTP::Get.new(uri)
+        when :post
+          req = Net::HTTP::Post.new(uri)
+          req["Content-Type"] = "application/json"
+          req.body = JSON.dump(payload) if payload
+          req
+        when :delete
+          Net::HTTP::Delete.new(uri)
+        else
+          raise ArgumentError, "Unsupported HTTP method: #{method}"
+        end
+
+        request["Authorization"] = "Bearer #{@state.api_key}"
+
+        # Execute request with timing
+        start_time = Time.now
+        Log.debug("[API] #{method.upcase} #{uri}")
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = (uri.scheme == "https")
+        response = http.request(request)
+
+        duration_ms = ((Time.now - start_time) * 1000).round(2)
+        Log.debug("[API] #{method.upcase} #{uri} -> #{response.code} (#{duration_ms}ms, #{response.body.bytesize} bytes)")
+
+        # Handle response
+        unless response.is_a?(Net::HTTPSuccess)
+          Log.debug("[API] Error response body: #{response.body}")
+          raise Error, "HTTP #{response.code} for #{method.upcase} #{uri}: #{response.body}"
+        end
+
+        parse_json ? JSON.parse(response.body) : response
+      end
+
+      # HTTP GET with query params - returns parsed JSON
+      def http_get(path, params = {})
+        http_request(:get, path, params: params)
+      end
+
+      # HTTP POST with JSON body - returns parsed JSON
+      def http_post_json(path, payload)
+        http_request(:post, path, payload: payload)
+      end
+
+      # HTTP DELETE - returns parsed JSON
+      def http_delete(path)
+        http_request(:delete, path)
+      end
+    end
+  end
+end

data/lib/braintrust/api/internal/auth.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "json"
+require "uri"
+require_relative "../../logger"
+
+module Braintrust
+  class API
+    module Internal
+      module Auth
+        # Result of a successful login
+        AuthResult = Struct.new(:org_id, :org_name, :api_url, :proxy_url, keyword_init: true)
+
+        # Mask API key for logging (show first 8 chars)
+        def self.mask_api_key(api_key)
+          return "nil" if api_key.nil?
+          return api_key if api_key.length <= 8
+          "#{api_key[0...8]}...#{api_key[-4..]}"
+        end
+
+        # Login to Braintrust API
+        # @param api_key [String] Braintrust API key
+        # @param app_url [String] Braintrust app URL
+        # @param org_name [String, nil] Optional org name to filter by
+        # @return [AuthResult] org info
+        # @raise [Braintrust::Error] if login fails
+        def self.login(api_key:, app_url:, org_name: nil)
+          masked_key = mask_api_key(api_key)
+          Log.debug("Login: attempting login with API key #{masked_key}, org #{org_name.inspect}, app URL #{app_url}")
+
+          uri = URI("#{app_url}/api/apikey/login")
+          request = Net::HTTP::Post.new(uri)
+          request["Authorization"] = "Bearer #{api_key}"
+
+          http = Net::HTTP.new(uri.hostname, uri.port)
+          http.use_ssl = true if uri.scheme == "https"
+
+          response = http.start do |http_session|
+            http_session.request(request)
+          end
+
+          Log.debug("Login: received response [#{response.code}]")
+
+          # Handle different status codes
+          case response
+          when Net::HTTPUnauthorized, Net::HTTPForbidden
+            raise Error, "Invalid API key: [#{response.code}]"
+          when Net::HTTPBadRequest
+            raise Error, "Bad request: [#{response.code}] #{response.body}"
+          when Net::HTTPClientError
+            raise Error, "Client error: [#{response.code}] #{response.message}"
+          when Net::HTTPServerError
+            raise Error, "Server error: [#{response.code}] #{response.message}"
+          when Net::HTTPSuccess
+            # Success - continue processing
+          else
+            raise Error, "Unexpected response: [#{response.code}] #{response.message}"
+          end
+
+          data = JSON.parse(response.body)
+          org_info_list = data["org_info"]
+
+          if org_info_list.nil? || org_info_list.empty?
+            raise Error, "No organizations found for API key"
+          end
+
+          # Select org: filter by org_name if present, else take first
+          org_info = if org_name
+            found = org_info_list.find { |org| org["name"] == org_name }
+            if found
+              Log.debug("Login: selected org '#{org_name}' (id: #{found["id"]})")
+              found
+            else
+              available = org_info_list.map { |o| o["name"] }.join(", ")
+              raise Error, "Organization '#{org_name}' not found. Available: #{available}"
+            end
+          else
+            selected = org_info_list.first
+            Log.debug("Login: selected first org '#{selected["name"]}' (id: #{selected["id"]})")
+            selected
+          end
+
+          result = AuthResult.new(
+            org_id: org_info["id"],
+            org_name: org_info["name"],
+            api_url: org_info["api_url"],
+            proxy_url: org_info["proxy_url"]
+          )
+
+          Log.debug("Login: successfully logged in as org '#{result.org_name}' (#{result.org_id})")
+          result
+        end
+      end
+    end
+  end
+end

data/lib/braintrust/api.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative "api/datasets"
+require_relative "api/functions"
+
+module Braintrust
+  # API client for Braintrust REST API
+  # Provides namespaced access to different API resources
+  class API
+    attr_reader :state
+
+    def initialize(state: nil)
+      @state = state || Braintrust.current_state
+      raise Error, "No state available" unless @state
+    end
+
+    # Access to datasets API
+    # @return [API::Datasets]
+    def datasets
+      @datasets ||= API::Datasets.new(self)
+    end
+
+    # Access to functions API
+    # @return [API::Functions]
+    def functions
+      @functions ||= API::Functions.new(self)
+    end
+  end
+end

data/lib/braintrust/config.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Braintrust
+  # Configuration object that reads from environment variables
+  # and allows overriding with explicit options
+  class Config
+    attr_reader :api_key, :org_name, :default_parent, :app_url, :api_url
+
+    def initialize(api_key: nil, org_name: nil, default_parent: nil, app_url: nil, api_url: nil)
+      @api_key = api_key
+      @org_name = org_name
+      @default_parent = default_parent
+      @app_url = app_url
+      @api_url = api_url
+    end
+
+    # Create a Config from environment variables, with option overrides
+    # Passed-in options take priority over ENV vars
+    def self.from_env(**options)
+      defaults = {
+        api_key: ENV["BRAINTRUST_API_KEY"],
+        org_name: ENV["BRAINTRUST_ORG_NAME"],
+        default_parent: ENV["BRAINTRUST_DEFAULT_PROJECT"],
+        app_url: ENV["BRAINTRUST_APP_URL"] || "https://www.braintrust.dev",
+        api_url: ENV["BRAINTRUST_API_URL"] || "https://api.braintrust.dev"
+      }
+      new(**defaults.merge(options))
+    end
+  end
+end

data/lib/braintrust/eval/case.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Braintrust
+  module Eval
+    # Case represents a single test case in an evaluation
+    # @attr input [Object] The input to the task
+    # @attr expected [Object, nil] The expected output (optional)
+    # @attr tags [Array<String>, nil] Optional tags for filtering/grouping
+    # @attr metadata [Hash, nil] Optional metadata for the case
+    Case = Struct.new(:input, :expected, :tags, :metadata, keyword_init: true)
+  end
+end

data/lib/braintrust/eval/cases.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require_relative "case"
+
+module Braintrust
+  module Eval
+    # Cases wraps test case data (arrays or enumerables) and normalizes them to Case objects
+    # Supports lazy evaluation for memory-efficient processing of large datasets
+    class Cases
+      include Enumerable
+
+      # Create a new Cases wrapper
+      # @param enumerable [Array, Enumerable] The test cases (hashes or Case objects)
+      def initialize(enumerable)
+        unless enumerable.respond_to?(:each)
+          raise ArgumentError, "Cases must be enumerable (respond to :each)"
+        end
+
+        @enumerable = enumerable
+      end
+
+      # Iterate over cases, normalizing each to a Case object
+      # @yield [Case] Each test case
+      def each
+        return enum_for(:each) unless block_given?
+
+        @enumerable.each do |item|
+          yield normalize_case(item)
+        end
+      end
+
+      # Get the count of cases
+      # Note: For lazy enumerators, this will force evaluation
+      # @return [Integer]
+      def count
+        @enumerable.count
+      end
+
+      private
+
+      # Normalize a case item to a Case object
+      # @param item [Hash, Case] The case item
+      # @return [Case]
+      def normalize_case(item)
+        case item
+        when Case
+          # Already a Case object
+          item
+        when Hash
+          # Convert hash to Case object
+          Case.new(**item)
+        else
+          raise ArgumentError, "Case must be a Hash or Case object, got #{item.class}"
+        end
+      end
+    end
+  end
+end