langsmith-sdk 0.1.1 → 0.3.0

data/lib/langsmith/client.rb

@@ -66,29 +66,6 @@ module Langsmith
       patch("/runs/#{run.id}", run.to_h, tenant_id: run.tenant_id)
     end
 
-    # Batch create/update runs.
-    # All runs in a batch should have the same tenant_id for optimal performance.
-    #
-    # @param post_runs [Array<Run>] runs to create
-    # @param patch_runs [Array<Run>] runs to update
-    # @param tenant_id [String, nil] tenant ID (inferred from runs if not provided)
-    # @return [Hash, nil] API response
-    # @raise [APIError] if the request fails
-    def batch_ingest(post_runs: [], patch_runs: [], tenant_id: nil)
-      return if post_runs.empty? && patch_runs.empty?
-
-      payload = {}
-      payload[:post] = post_runs.map(&:to_h) unless post_runs.empty?
-      payload[:patch] = patch_runs.map(&:to_h) unless patch_runs.empty?
-
-      # Use tenant_id from first run if not explicitly provided
-      effective_tenant_id = tenant_id ||
-                            post_runs.first&.tenant_id ||
-                            patch_runs.first&.tenant_id
-
-      post("/runs/batch", payload, tenant_id: effective_tenant_id)
-    end
-
     # Batch create/update runs using pre-serialized hashes.
     # Used by BatchProcessor which snapshots run data at enqueue time.
     #
@@ -97,7 +74,7 @@ module Langsmith
     # @param tenant_id [String, nil] tenant ID for the request
     # @return [Hash, nil] API response
     # @raise [APIError] if the request fails
-    def batch_ingest_raw(post_runs: [], patch_runs: [], tenant_id: nil)
+    def batch_ingest(post_runs: [], patch_runs: [], tenant_id: nil)
       return if post_runs.empty? && patch_runs.empty?
 
       payload = {}
@@ -107,8 +84,102 @@ module Langsmith
       post("/runs/batch", payload, tenant_id: tenant_id)
     end
 
+    # List examples from a LangSmith dataset.
+    #
+    # @param dataset_id [String] the dataset ID to fetch examples from
+    # @param limit [Integer, nil] max number of examples to return (API max: 100)
+    # @param offset [Integer, nil] number of examples to skip
+    # @param tenant_id [String, nil] tenant ID (falls back to configured tenant_id)
+    # @return [Array<Hash>] array of example objects
+    # @raise [APIError] if the request fails
+    def list_examples(dataset_id:, limit: nil, offset: nil, tenant_id: nil)
+      params = { dataset: dataset_id }
+      params[:limit] = limit if limit
+      params[:offset] = offset if offset
+
+      get("/api/v1/examples", params: params, tenant_id: resolve_tenant_id(tenant_id))
+    end
+
+    # Create a new experiment (tracer session) linked to a dataset.
+    #
+    # @param name [String] experiment name
+    # @param dataset_id [String] reference dataset ID
+    # @param description [String, nil] optional experiment description
+    # @param metadata [Hash, nil] optional metadata (stored as `extra`)
+    # @param tenant_id [String, nil] tenant ID (falls back to configured tenant_id)
+    # @return [Hash] the created experiment object
+    # @raise [APIError] if the request fails
+    def create_experiment(name:, dataset_id:, description: nil, metadata: nil, tenant_id: nil)
+      payload = {
+        name: name,
+        reference_dataset_id: dataset_id,
+        start_time: Time.now.utc.iso8601
+      }
+      payload[:description] = description if description
+      payload[:extra] = metadata if metadata
+
+      post("/api/v1/sessions", payload, tenant_id: resolve_tenant_id(tenant_id))
+    end
+
+    # Close an experiment by setting its end time.
+    #
+    # @param experiment_id [String] the experiment (session) ID
+    # @param end_time [String] ISO-8601 end time
+    # @param tenant_id [String, nil] tenant ID (falls back to configured tenant_id)
+    # @return [Hash] the updated experiment object
+    # @raise [APIError] if the request fails
+    def close_experiment(experiment_id:, end_time:, tenant_id: nil)
+      patch("/api/v1/sessions/#{experiment_id}", { end_time: end_time }, tenant_id: resolve_tenant_id(tenant_id))
+    end
+
+    # Create feedback (a score/annotation) on a run.
+    #
+    # @param run_id [String] UUID of the run to attach feedback to
+    # @param key [String] metric name (e.g. "correctness")
+    # @param score [Numeric, nil] numeric score (typically 0.0-1.0)
+    # @param value [String, nil] categorical value (alternative to score)
+    # @param comment [String, nil] explanation or reasoning
+    # @param tenant_id [String, nil] tenant ID (falls back to configured tenant_id)
+    # @return [Hash] the created feedback object
+    # @raise [APIError] if the request fails
+    def create_feedback(run_id:, key:, score: nil, value: nil, comment: nil, tenant_id: nil)
+      payload = { run_id: run_id, key: key }
+      payload[:score] = score if score
+      payload[:value] = value if value
+      payload[:comment] = comment if comment
+
+      post("/api/v1/feedback", payload, tenant_id: resolve_tenant_id(tenant_id))
+    end
+
+    # Read a single run by ID.
+    #
+    # @param run_id [String] UUID of the run to fetch
+    # @param tenant_id [String, nil] tenant ID (falls back to configured tenant_id)
+    # @return [Hash] the run object (inputs, outputs, child_run_ids, tokens, etc.)
+    # @raise [APIError] if the request fails
+    def read_run(run_id:, tenant_id: nil)
+      get("/api/v1/runs/#{run_id}", tenant_id: resolve_tenant_id(tenant_id))
+    end
+
     private
 
+    def resolve_tenant_id(tenant_id)
+      tenant_id || Langsmith.configuration.tenant_id
+    end
+
+    def get(path, params: {}, tenant_id: nil)
+      response = connection.get(path, params) do |req|
+        req.headers["X-Tenant-Id"] = tenant_id if tenant_id
+      end
+      handle_response(response)
+    rescue Faraday::ConnectionFailed, Faraday::TimeoutError => e
+      raise APIError, "Network error: #{e.message}"
+    rescue Faraday::Error => e
+      raise APIError, "Request failed: #{e.message}" unless e.respond_to?(:response) && e.response
+
+      handle_response(e.response)
+    end
+
     def connection
       @connection ||= Faraday.new(url: @endpoint) do |f|
         f.request :json

data/lib/langsmith/configuration.rb

@@ -42,6 +42,9 @@ module Langsmith
     # @return [String, nil] Tenant ID for multi-tenant scenarios
     attr_accessor :tenant_id
 
+    # @return [Integer, nil] Maximum buffered run entries (queue + pending); nil means unlimited
+    attr_accessor :max_pending_entries
+
     def initialize
       @api_key = ENV.fetch("LANGSMITH_API_KEY", nil)
       @endpoint = ENV.fetch("LANGSMITH_ENDPOINT", "https://api.smith.langchain.com")
@@ -52,6 +55,7 @@ module Langsmith
       @timeout = ENV.fetch("LANGSMITH_TIMEOUT", 10).to_i
       @max_retries = ENV.fetch("LANGSMITH_MAX_RETRIES", 3).to_i
       @tenant_id = ENV.fetch("LANGSMITH_TENANT_ID", nil)
+      @max_pending_entries = ENV.fetch("LANGSMITH_MAX_PENDING_ENTRIES", nil)&.to_i
     end
 
     # Returns whether tracing is enabled in configuration.

data/lib/langsmith/context.rb

@@ -12,7 +12,9 @@ module Langsmith
   # and caused test failures on Ruby 3.2.
   module Context
     CONTEXT_KEY = :langsmith_run_stack
-    private_constant :CONTEXT_KEY
+    EVALUATION_CONTEXT_KEY = :langsmith_evaluation_context
+    EVALUATION_ROOT_RUN_ID_KEY = :langsmith_evaluation_root_run_id
+    private_constant :CONTEXT_KEY, :EVALUATION_CONTEXT_KEY, :EVALUATION_ROOT_RUN_ID_KEY
 
     class << self
       # Returns the current run stack for this thread.
@@ -49,9 +51,11 @@ module Langsmith
         pop
       end
 
-      # Clear the entire run stack (useful for testing)
+      # Clear the entire run stack and evaluation context (useful for testing)
       def clear!
         Thread.current[CONTEXT_KEY] = []
+        Thread.current[EVALUATION_CONTEXT_KEY] = nil
+        Thread.current[EVALUATION_ROOT_RUN_ID_KEY] = nil
       end
 
       # Check if there's an active trace context
@@ -68,6 +72,45 @@ module Langsmith
       def root_run
         run_stack.first
       end
+
+      # Returns the current evaluation context, or nil when not in evaluation.
+      # @return [Hash, nil] hash with :experiment_id and :example_id, or nil
+      def evaluation_context
+        Thread.current[EVALUATION_CONTEXT_KEY]
+      end
+
+      # Returns true when evaluation context is set.
+      # @return [Boolean]
+      def evaluating?
+        !evaluation_context.nil?
+      end
+
+      # Stores the root run ID for the current evaluation example.
+      # Called by RunTree when creating the first root run inside an evaluation block.
+      #
+      # @param run_id [String] the root run's ID
+      def set_evaluation_root_run_id(run_id)
+        Thread.current[EVALUATION_ROOT_RUN_ID_KEY] = run_id
+      end
+
+      # Returns the root run ID for the current evaluation example, or nil.
+      # @return [String, nil]
+      def evaluation_root_run_id
+        Thread.current[EVALUATION_ROOT_RUN_ID_KEY]
+      end
+
+      # Execute a block with evaluation context set.
+      # Context is cleared in ensure block even if the block raises.
+      #
+      # @param experiment_id [String] the experiment session ID
+      # @param example_id [String] the dataset example ID
+      def with_evaluation(experiment_id:, example_id:)
+        Thread.current[EVALUATION_CONTEXT_KEY] = { experiment_id: experiment_id, example_id: example_id }
+        yield
+      ensure
+        Thread.current[EVALUATION_CONTEXT_KEY] = nil
+        Thread.current[EVALUATION_ROOT_RUN_ID_KEY] = nil
+      end
     end
   end
 end

data/lib/langsmith/evaluation/experiment_runner.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Langsmith
+  module Evaluation
+    # Orchestrates running an evaluation experiment against a dataset.
+    #
+    # Fetches examples, creates an experiment, runs each example through
+    # the user-provided block with evaluation context set, scores outputs
+    # with evaluators, and returns a summary of results.
+    class ExperimentRunner
+      # @param dataset_id [String] the dataset to evaluate against
+      # @param experiment_name [String] name for the experiment
+      # @param description [String, nil] optional experiment description
+      # @param metadata [Hash, nil] optional experiment metadata
+      # @param evaluators [Hash] map of evaluator key to callable
+      # @param block [Proc] block that receives each example and produces a result
+      def initialize(dataset_id:, experiment_name:, description: nil, metadata: nil, evaluators: {}, &block)
+        @dataset_id = dataset_id
+        @experiment_name = experiment_name
+        @description = description
+        @metadata = metadata
+        @evaluators = evaluators
+        @block = block
+      end
+
+      # Run the evaluation experiment.
+      #
+      # @return [Hash] summary with :experiment_id, :total, :succeeded, :failed, :results
+      def run
+        examples = client.list_examples(dataset_id: @dataset_id)
+
+        experiment = client.create_experiment(
+          name: @experiment_name,
+          dataset_id: @dataset_id,
+          description: @description,
+          metadata: @metadata
+        )
+        experiment_id = experiment[:id]
+
+        results = examples.map { |example| run_example(example, experiment_id) }
+
+        Langsmith.flush
+        client.close_experiment(experiment_id: experiment_id, end_time: Time.now.utc.iso8601)
+
+        build_summary(experiment_id, results)
+      end
+
+      private
+
+      def client
+        Langsmith.client
+      end
+
+      def run_example(example, experiment_id)
+        outputs = nil
+        run_id = nil
+
+        begin
+          Context.with_evaluation(experiment_id: experiment_id, example_id: example[:id]) do
+            outputs = @block.call(example)
+            run_id = Context.evaluation_root_run_id
+          end
+        rescue StandardError => e
+          return { example_id: example[:id], run_id: nil, status: :error, error: e.message, feedback: nil }
+        end
+
+        feedback = run_evaluators(example, outputs, run_id)
+        { example_id: example[:id], run_id: run_id, status: :success, error: nil, feedback: feedback }
+      rescue StandardError => e
+        { example_id: example[:id], run_id: run_id, status: :success, error: e.message, feedback: nil }
+      end
+
+      def run_evaluators(example, outputs, run_id)
+        return nil if @evaluators.empty? || run_id.nil?
+
+        Langsmith.flush
+        run = fetch_run_with_retry(run_id)
+
+        @evaluators.each_with_object({}) do |(key, evaluator), feedback|
+          feedback[key] = execute_evaluator(key, evaluator, example, outputs, run_id, run)
+        end
+      end
+
+      # LangSmith has indexing lag after batch ingest — the run may not be
+      # queryable immediately. Retry a few times with a short delay.
+      def fetch_run_with_retry(run_id, retries: 3, delay: 1)
+        client.read_run(run_id: run_id)
+      rescue Client::APIError => e
+        raise unless e.status_code == 404 && retries.positive?
+
+        sleep(delay)
+        fetch_run_with_retry(run_id, retries: retries - 1, delay: delay)
+      end
+
+      def execute_evaluator(key, evaluator, example, outputs, run_id, run)
+        result = evaluator.call(
+          outputs: outputs,
+          reference_outputs: example[:outputs],
+          inputs: example[:inputs],
+          run: run
+        )
+        return { score: nil, success: true, skipped: true } if result.nil?
+
+        normalized = normalize_result(result)
+        client.create_feedback(run_id: run_id, key: key.to_s, **normalized)
+        normalized.merge(success: true)
+      rescue StandardError => e
+        { score: nil, success: false, error: e.message }
+      end
+
+      def normalize_result(result)
+        case result
+        when true  then { score: 1.0, value: nil, comment: nil }
+        when false then { score: 0.0, value: nil, comment: nil }
+        when Hash  then { score: result[:score], value: result[:value], comment: result[:comment] }
+        else            { score: result, value: nil, comment: nil }
+        end
+      end
+
+      def build_summary(experiment_id, results)
+        {
+          experiment_id: experiment_id,
+          total: results.size,
+          succeeded: results.count { |r| r[:status] == :success },
+          failed: results.count { |r| r[:status] == :error },
+          results: results
+        }
+      end
+    end
+  end
+end

data/lib/langsmith/evaluation.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "evaluation/experiment_runner"
+
+module Langsmith
+  # Public API for running evaluations against LangSmith datasets.
+  #
+  # @example
+  #   Langsmith::Evaluation.run(
+  #     dataset_id: "dataset-uuid",
+  #     experiment_name: "my-experiment"
+  #   ) do |example|
+  #     Langsmith.trace("eval", run_type: "chain", inputs: example[:inputs]) do
+  #       my_app.call(example[:inputs])
+  #     end
+  #   end
+  module Evaluation
+    # Run an evaluation experiment against a dataset.
+    #
+    # @param dataset_id [String] the dataset to evaluate against
+    # @param experiment_name [String] name for the experiment
+    # @param description [String, nil] optional experiment description
+    # @param metadata [Hash, nil] optional experiment metadata
+    # @param evaluators [Hash] map of evaluator key to callable (see ExperimentRunner)
+    # @yield [Hash] each dataset example
+    # @return [Hash] summary with :experiment_id, :total, :succeeded, :failed, :results
+    def self.run(dataset_id:, experiment_name:, description: nil, metadata: nil, evaluators: {}, &block)
+      ExperimentRunner.new(
+        dataset_id: dataset_id,
+        experiment_name: experiment_name,
+        description: description,
+        metadata: metadata,
+        evaluators: evaluators,
+        &block
+      ).run
+    end
+  end
+end

data/lib/langsmith/run.rb

@@ -37,6 +37,12 @@ module Langsmith
     # @return [String, nil] tenant ID for multi-tenant scenarios
     attr_reader :tenant_id
 
+    # @return [String, nil] links this run to a dataset example (for evaluations)
+    attr_reader :reference_example_id
+
+    # @return [String, nil] links this run to an experiment session (overrides project routing)
+    attr_reader :session_id
+
     # @return [String] trace ID (root run's ID)
     attr_reader :trace_id
 
@@ -81,6 +87,8 @@ module Langsmith
     # @param tenant_id [String, nil] tenant ID for multi-tenant scenarios
     # @param trace_id [String, nil] trace ID (defaults to own ID for root runs)
     # @param parent_dotted_order [String, nil] parent's dotted order for tree ordering
+    # @param reference_example_id [String, nil] dataset example ID (for evaluations)
+    # @param session_id [String, nil] experiment session ID (overrides project routing)
     #
     # @raise [ArgumentError] if run_type is invalid
     def initialize(
@@ -95,7 +103,9 @@ module Langsmith
       id: nil,
       tenant_id: nil,
       trace_id: nil,
-      parent_dotted_order: nil
+      parent_dotted_order: nil,
+      reference_example_id: nil,
+      session_id: nil
     )
       @id = id || SecureRandom.uuid
       @name = name
@@ -114,6 +124,8 @@ module Langsmith
       @tags = tags || []
       @extra = extra || {}
       @events = []
+      @reference_example_id = reference_example_id
+      @session_id = session_id
       # dotted_order is used for ordering runs in the trace tree
       @dotted_order = build_dotted_order(parent_dotted_order)
     end
@@ -242,6 +254,8 @@ module Langsmith
         outputs:,
         error:,
         parent_run_id:,
+        reference_example_id:,
+        session_id:,
         trace_id:,
         dotted_order:,
         session_name:,

data/lib/langsmith/run_tree.rb

@@ -20,23 +20,19 @@ module Langsmith
       tenant_id: nil,
       project: nil
     )
-      # If no explicit parent, check context for current parent
       effective_parent_id = parent_run_id || Context.current_parent_run_id
-
-      # Inherit tenant_id from parent run if not explicitly set
       effective_tenant_id = tenant_id || Context.current_run&.tenant_id
 
       # Child traces must use the same project as their parent to keep the trace tree together.
-      # Only root traces can set the project; children always inherit from parent.
       effective_project = Context.current_run&.session_name || project
-
-      # Inherit trace_id from root run (parent's trace_id)
-      # For root runs, trace_id will default to the run's own ID
       effective_trace_id = Context.current_run&.trace_id
-
-      # Inherit dotted_order from parent for proper trace ordering
       parent_dotted_order = Context.current_run&.dotted_order
 
+      # Inject evaluation context: session_id on ALL runs, reference_example_id only on root runs.
+      eval_ctx = Context.evaluation_context
+      effective_session_id = eval_ctx&.dig(:experiment_id)
+      effective_ref_example_id = eval_ctx&.dig(:example_id) unless effective_parent_id
+
       @run = Run.new(
         name: name,
         run_type: run_type,
@@ -48,11 +44,13 @@ module Langsmith
         tenant_id: effective_tenant_id,
         session_name: effective_project,
         trace_id: effective_trace_id,
-        parent_dotted_order: parent_dotted_order
+        parent_dotted_order: parent_dotted_order,
+        reference_example_id: effective_ref_example_id,
+        session_id: effective_session_id
       )
 
-      @posted_start = false
-      @posted_end = false
+      register_evaluation_root_run(effective_parent_id)
+      @posted_start = @posted_end = false
     end
 
     # Post the run start to LangSmith
@@ -138,6 +136,13 @@ module Langsmith
 
     private
 
+    # Register the root run ID in evaluation context so the runner can
+    # attach feedback to it later. Only root runs (no parent) register;
+    # child runs must not overwrite.
+    def register_evaluation_root_run(effective_parent_id)
+      Context.set_evaluation_root_run_id(@run.id) if effective_parent_id.nil? && Context.evaluating?
+    end
+
     # Sanitize block results to prevent circular references.
     # When users call methods like `run.add_metadata(...)` as the last line,
     # the Run object itself becomes the result, creating a circular reference.

data/lib/langsmith/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Langsmith
-  VERSION = "0.1.1"
+  VERSION = "0.3.0"
 end

data/lib/langsmith.rb

@@ -8,7 +8,7 @@ require_relative "langsmith/context"
 require_relative "langsmith/client"
 require_relative "langsmith/batch_processor"
 require_relative "langsmith/run_tree"
-require_relative "langsmith/traceable"
+require_relative "langsmith/evaluation"
 
 module Langsmith
   class << self

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: langsmith-sdk
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Felipe Cabezudo
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2025-12-21 00:00:00.000000000 Z
+date: 2026-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -91,17 +91,17 @@ files:
 - examples/complex_agent.rb
 - examples/llm_tracing.rb
 - examples/openai_integration.rb
-- langsmith.gemspec
 - lib/langsmith.rb
 - lib/langsmith/batch_processor.rb
 - lib/langsmith/client.rb
 - lib/langsmith/configuration.rb
 - lib/langsmith/context.rb
 - lib/langsmith/errors.rb
+- lib/langsmith/evaluation.rb
+- lib/langsmith/evaluation/experiment_runner.rb
 - lib/langsmith/railtie.rb
 - lib/langsmith/run.rb
 - lib/langsmith/run_tree.rb
-- lib/langsmith/traceable.rb
 - lib/langsmith/version.rb
 homepage: https://github.com/felipekb/langsmith-ruby-sdk
 licenses:
@@ -127,7 +127,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.17
+rubygems_version: 3.4.19
 signing_key:
 specification_version: 4
 summary: Ruby SDK for LangSmith tracing and observability

data/langsmith.gemspec

@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "lib/langsmith/version"
-
-Gem::Specification.new do |spec|
-  spec.name = "langsmith-sdk"
-  spec.version = Langsmith::VERSION
-  spec.authors = ["Felipe Cabezudo"]
-  spec.email = ["felipecabedilo@gmail.com"]
-
-  spec.summary = "Ruby SDK for LangSmith tracing and observability"
-  spec.description = "A Ruby client for LangSmith, providing tracing and observability for LLM applications"
-  spec.homepage = "https://github.com/felipekb/langsmith-ruby-sdk"
-  spec.license = "MIT"
-  spec.required_ruby_version = ">= 3.1.0"
-
-  spec.metadata["allowed_push_host"] = "https://rubygems.org"
-  spec.metadata["homepage_uri"] = spec.homepage
-  spec.metadata["source_code_uri"] = spec.homepage
-  spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/main/CHANGELOG.md"
-  spec.metadata["rubygems_mfa_required"] = "true"
-
-  spec.files = Dir.chdir(__dir__) do
-    `git ls-files -z`.split("\x0").reject do |f|
-      (File.expand_path(f) == __FILE__) ||
-        f.start_with?(*%w[bin/ test/ spec/ features/ .git .github appveyor Gemfile])
-    end
-  end
-  spec.bindir = "exe"
-  spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
-  spec.require_paths = ["lib"]
-
-  # Runtime dependencies
-  spec.add_dependency "concurrent-ruby", ">= 1.1", "< 3.0"
-  spec.add_dependency "faraday", "~> 2.0"
-  spec.add_dependency "faraday-net_http_persistent", "~> 2.0"
-  spec.add_dependency "faraday-retry", "~> 2.0"
-end