langsmithrb 0.1.0

data/lib/langsmith/dataset.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Langsmith
+  class Dataset
+    attr_reader :id, :name, :description, :created_at, :tenant_id, :data
+
+    # Initialize a new Dataset instance
+    #
+    # @param client [Langsmith::Client] The LangSmith client
+    # @param data [Hash] Dataset data from the API
+    def initialize(client, data)
+      @client = client
+      @id = data[:id] || data["id"]
+      @name = data[:name] || data["name"]
+      @description = data[:description] || data["description"]
+      created_at_value = data[:created_at] || data["created_at"]
+      @created_at = created_at_value ? Time.parse(created_at_value) : nil
+      @tenant_id = data[:tenant_id] || data["tenant_id"]
+      @data = data
+    end
+
+    # Create a new example in this dataset
+    #
+    # @param inputs [Hash] Input values for the example
+    # @param outputs [Hash, nil] Output values for the example (optional)
+    # @param metadata [Hash, nil] Additional metadata for the example (optional)
+    # @return [Langsmith::Example] The created example
+    def create_example(inputs:, outputs: nil, metadata: nil)
+      data = {
+        dataset_id: @id,
+        inputs: inputs
+      }
+      data[:outputs] = outputs if outputs
+      data[:metadata] = metadata if metadata
+      
+      response = @client.post("/examples", data)
+      Example.new(@client, response)
+    end
+
+    # Get a specific example by ID
+    #
+    # @param example_id [String] ID of the example to get
+    # @return [Langsmith::Example] The requested example
+    def get_example(example_id:)
+      response = @client.get("/examples/#{example_id}")
+      Example.new(@client, response)
+    end
+
+    # List examples in this dataset
+    #
+    # @param limit [Integer] Maximum number of examples to return
+    # @param offset [Integer] Number of examples to skip
+    # @return [Array<Langsmith::Example>] List of examples in this dataset
+    def list_examples(limit: 100, offset: 0)
+      params = {
+        dataset_id: @id,
+        limit: limit,
+        offset: offset
+      }
+      response = @client.get("/examples", params)
+      response.map { |example_data| Example.new(@client, example_data) }
+    end
+    
+    # Create multiple examples in batch
+    #
+    # @param examples [Array<Hash>] Array of example data, each containing :inputs and optionally :outputs and :metadata
+    # @return [Array<Example>] The created examples
+    def create_examples_batch(examples:)
+      @client.create_examples_batch(dataset_id: @id, examples: examples)
+    end
+
+    # Create a new evaluation run on this dataset
+    #
+    # @param evaluator_name [String] Name of the evaluator to use
+    # @param run_ids [Array<String>] IDs of runs to evaluate
+    # @param metadata [Hash, nil] Additional metadata for the evaluation (optional)
+    # @return [Langsmith::Evaluation] The created evaluation run
+    def create_evaluation_run(evaluator_name:, run_ids:, metadata: nil)
+      data = {
+        dataset_id: @id,
+        evaluator_name: evaluator_name,
+        run_ids: run_ids
+      }
+      data[:metadata] = metadata if metadata
+      
+      response = @client.post("/evaluations", data)
+      Evaluation.new(@client, response)
+    end
+
+    # List evaluation runs for this dataset
+    #
+    # @param limit [Integer] Maximum number of evaluation runs to return
+    # @param offset [Integer] Number of evaluation runs to skip
+    # @return [Array<Langsmith::Evaluation>] List of evaluation runs for this dataset
+    def list_evaluation_runs(limit: 100, offset: 0)
+      params = {
+        dataset_id: @id,
+        limit: limit,
+        offset: offset
+      }
+      response = @client.get("/evaluations", params)
+      response.map { |eval_data| Evaluation.new(@client, eval_data) }
+    end
+
+    # Update this dataset
+    #
+    # @param name [String, nil] New name for the dataset (optional)
+    # @param description [String, nil] New description for the dataset (optional)
+    # @return [Langsmith::Dataset] The updated dataset
+    def update(name: nil, description: nil)
+      data = {}
+      data[:name] = name if name
+      data[:description] = description if description
+      
+      response = @client.patch("/datasets/#{@id}", data)
+      @name = response[:name] || response["name"] if name
+      @description = response[:description] || response["description"] if description
+      @data = response
+      self
+    end
+
+    # Delete this dataset
+    #
+    # @return [Boolean] True if successful
+    def delete
+      @client.delete("/datasets/#{@id}")
+      true
+    end
+  end
+
+  # Example class to represent dataset examples
+  class Example
+    attr_reader :id, :dataset_id, :inputs, :outputs, :metadata, :created_at
+
+    def initialize(client, data)
+      @client = client
+      @id = data[:id] || data["id"]
+      @dataset_id = data[:dataset_id] || data["dataset_id"]
+      @inputs = data[:inputs] || data["inputs"]
+      @outputs = data[:outputs] || data["outputs"]
+      @metadata = data[:metadata] || data["metadata"]
+      created_at_value = data[:created_at] || data["created_at"]
+      @created_at = created_at_value ? Time.parse(created_at_value) : nil
+      @data = data
+    end
+
+    # Update this example
+    #
+    # @param inputs [Hash, nil] New input values (optional)
+    # @param outputs [Hash, nil] New output values (optional)
+    # @param metadata [Hash, nil] New metadata (optional)
+    # @return [Langsmith::Example] The updated example
+    def update(inputs: nil, outputs: nil, metadata: nil)
+      data = {}
+      data[:inputs] = inputs if inputs
+      data[:outputs] = outputs if outputs
+      data[:metadata] = metadata if metadata
+      
+      response = @client.patch("/examples/#{@id}", data)
+      @inputs = response[:inputs] || response["inputs"] if inputs
+      @outputs = response[:outputs] || response["outputs"] if outputs
+      @metadata = response[:metadata] || response["metadata"] if metadata
+      @data = response
+      self
+    end
+
+    # Delete this example
+    #
+    # @return [Boolean] True if successful
+    def delete
+      @client.delete("/examples/#{@id}")
+      true
+    end
+  end
+end

data/lib/langsmith/evaluation.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Langsmith
+  class Evaluation
+    attr_reader :id, :dataset_id, :evaluator_name, :status, :created_at, :run_ids, :metadata, :data
+
+    # Initialize a new Evaluation instance
+    #
+    # @param client [Langsmith::Client] The LangSmith client
+    # @param data [Hash] Evaluation data from the API
+    def initialize(client, data)
+      @client = client
+      @id = data[:id] || data["id"]
+      @dataset_id = data[:dataset_id] || data["dataset_id"]
+      @evaluator_name = data[:evaluator_name] || data["evaluator_name"]
+      @status = data[:status] || data["status"]
+      created_at_value = data[:created_at] || data["created_at"]
+      @created_at = created_at_value ? Time.parse(created_at_value) : nil
+      @run_ids = data[:run_ids] || data["run_ids"] || []
+      @metadata = data[:metadata] || data["metadata"] || {}
+      @data = data
+    end
+
+    # Get the results of this evaluation
+    #
+    # @param limit [Integer] Maximum number of results to return
+    # @param offset [Integer] Number of results to skip
+    # @return [Array<Hash>] Evaluation results
+    def results(limit: 100, offset: 0)
+      params = {
+        limit: limit,
+        offset: offset
+      }
+      @client.get("/evaluations/#{@id}/results", params)
+    end
+
+    # Get the status of this evaluation
+    #
+    # @return [String] Current status of the evaluation
+    def refresh_status
+      response = @client.get("/evaluations/#{@id}")
+      @status = response[:status] || response["status"]
+      @data = response
+      @status
+    end
+
+    # Check if the evaluation is completed
+    #
+    # @return [Boolean] True if the evaluation is completed
+    def completed?
+      refresh_status == "complete"
+    end
+
+    # Wait for the evaluation to complete
+    #
+    # @param timeout [Integer] Maximum time to wait in seconds
+    # @param interval [Integer] Time between status checks in seconds
+    # @return [Boolean] True if the evaluation completed within the timeout
+    def wait_for_completion(timeout: 300, interval: 5)
+      start_time = Time.now
+      
+      while Time.now - start_time < timeout
+        return true if completed?
+        sleep(interval)
+      end
+      
+      false
+    end
+    
+    # Update this evaluation's metadata
+    #
+    # @param metadata [Hash] New metadata for the evaluation
+    # @return [Langsmith::Evaluation] The updated evaluation
+    def update_metadata(metadata:)
+      data = { metadata: metadata }
+      response = @client.patch("/evaluations/#{@id}", data)
+      @metadata = response[:metadata] || response["metadata"]
+      @data = response
+      self
+    end
+    
+    # Cancel this evaluation
+    #
+    # @return [Boolean] True if successful
+    def cancel
+      @client.post("/evaluations/#{@id}/cancel")
+      refresh_status
+      true
+    end
+    
+    # Delete this evaluation
+    #
+    # @return [Boolean] True if successful
+    def delete
+      @client.delete("/evaluations/#{@id}")
+      true
+    end
+  end
+end

data/lib/langsmith/feedback.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Langsmith
+  class Feedback
+    attr_reader :id, :run_id, :key, :score, :comment, :created_at
+
+    # Initialize a new Feedback instance
+    #
+    # @param client [Langsmith::Client] The LangSmith client
+    # @param data [Hash] Feedback data from the API
+    def initialize(client, data)
+      @client = client
+      @id = data["id"]
+      @run_id = data["run_id"]
+      @key = data["key"]
+      @score = data["score"]
+      @comment = data["comment"]
+      @created_at = data["created_at"] ? Time.parse(data["created_at"]) : nil
+    end
+
+    # Update this feedback
+    #
+    # @param score [Float] New feedback score
+    # @param comment [String] New feedback comment
+    # @return [Langsmith::Feedback] The updated feedback
+    def update(score: nil, comment: nil)
+      data = {}
+      data[:score] = score if score
+      data[:comment] = comment if comment
+
+      response = @client.patch("/feedback/#{@id}", data)
+      Langsmith::Feedback.new(@client, response)
+    end
+
+    # Delete this feedback
+    #
+    # @return [Boolean] True if the feedback was deleted successfully
+    def delete
+      response = @client.delete("/feedback/#{@id}")
+      response["success"] == true
+    end
+  end
+end

data/lib/langsmith/project.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Langsmith
+  class Project
+    attr_reader :id, :name, :description, :created_at, :tenant_id
+
+    # Initialize a new Project instance
+    #
+    # @param client [Langsmith::Client] The LangSmith client
+    # @param data [Hash] Project data from the API
+    def initialize(client, data)
+      @client = client
+      @id = data["id"]
+      @name = data["name"]
+      @description = data["description"]
+      @created_at = data["created_at"] ? Time.parse(data["created_at"]) : nil
+      @tenant_id = data["tenant_id"]
+    end
+
+    # Create a new run in this project
+    #
+    # @param name [String] Name of the run
+    # @param run_type [String] Type of run (e.g., llm, chain, tool)
+    # @param inputs [Hash] Input values for the run
+    # @param extra [Hash] Additional metadata for the run
+    # @return [Langsmith::Run] The created run
+    def create_run(name:, run_type:, inputs: {}, extra: {})
+      @client.create_run(name: name, run_type: run_type, project_name: @name, inputs: inputs, extra: extra)
+    end
+
+    # List runs in this project
+    #
+    # @param run_type [String] Filter by run type
+    # @param limit [Integer] Maximum number of runs to return
+    # @return [Array<Langsmith::Run>] List of runs in this project
+    def list_runs(run_type: nil, limit: 100)
+      @client.list_runs(project_name: @name, run_type: run_type, limit: limit)
+    end
+  end
+end

data/lib/langsmith/run.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Langsmith
+  class Run
+    attr_reader :id, :name, :run_type, :start_time, :end_time, :status
+    attr_reader :inputs, :outputs, :error, :project_name, :trace_id, :parent_run_id
+
+    # Initialize a new Run instance
+    #
+    # @param client [Langsmith::Client] The LangSmith client
+    # @param data [Hash] Run data from the API
+    def initialize(client, data)
+      @client = client
+      @id = data["id"]
+      @name = data["name"]
+      @run_type = data["run_type"]
+      @start_time = data["start_time"] ? Time.parse(data["start_time"]) : Time.now
+      @end_time = data["end_time"] ? Time.parse(data["end_time"]) : nil
+      @status = data["status"] || "in_progress"
+      @inputs = data["inputs"] || {}
+      @outputs = data["outputs"] || {}
+      @error = data["error"]
+      @project_name = data["project_name"]
+      @trace_id = data["trace_id"]
+      @parent_run_id = data["parent_run_id"]
+      @extra = data["extra"] || {}
+    end
+
+    # Update the run with outputs and mark it as completed
+    #
+    # @param outputs [Hash] Output values from the run
+    # @return [Langsmith::Run] The updated run
+    def end(outputs: nil, error: nil)
+      end_time = Time.now
+      
+      if error
+        @client.update_run(run_id: @id, end_time: end_time, error: error)
+      else
+        @client.update_run(run_id: @id, outputs: outputs, end_time: end_time)
+      end
+    end
+
+    # Create a child run for this run
+    #
+    # @param name [String] Name of the child run
+    # @param run_type [String] Type of the child run
+    # @param inputs [Hash] Input values for the child run
+    # @param extra [Hash] Additional metadata for the child run
+    # @return [Langsmith::Run] The created child run
+    def create_child_run(name:, run_type:, inputs: {}, extra: {})
+      child_extra = extra.merge(parent_run_id: @id, trace_id: @trace_id)
+      
+      data = {
+        name: name,
+        run_type: run_type,
+        inputs: inputs,
+        extra: child_extra
+      }
+      data[:project_name] = @project_name if @project_name
+
+      response = @client.post("/runs", data)
+      Langsmith::Run.new(@client, response)
+    end
+
+    # Add feedback to this run
+    #
+    # @param key [String] Feedback key (e.g., "correctness", "helpfulness")
+    # @param score [Float] Feedback score (typically 0.0 to 1.0)
+    # @param comment [String] Optional comment with the feedback
+    # @return [Langsmith::Feedback] The created feedback
+    def add_feedback(key:, score:, comment: nil)
+      @client.create_feedback(run_id: @id, key: key, score: score, comment: comment)
+    end
+
+    # Get all feedback for this run
+    #
+    # @return [Array<Langsmith::Feedback>] List of feedback for this run
+    def get_feedback
+      @client.get_feedback(run_id: @id)
+    end
+
+    # Get metadata from the run
+    #
+    # @param key [String] The metadata key to retrieve
+    # @return [Object] The metadata value
+    def get_metadata(key)
+      @extra[key]
+    end
+
+    # Check if the run is completed
+    #
+    # @return [Boolean] True if the run is completed
+    def completed?
+      !@end_time.nil?
+    end
+
+    # Check if the run has an error
+    #
+    # @return [Boolean] True if the run has an error
+    def error?
+      !@error.nil?
+    end
+
+    # Get the duration of the run in seconds
+    #
+    # @return [Float] Duration in seconds, or nil if the run is not completed
+    def duration
+      return nil unless @end_time
+      @end_time - @start_time
+    end
+  end
+end

data/lib/langsmith/trace.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module Langsmith
+  class Trace
+    attr_reader :id, :name, :start_time, :end_time, :runs
+
+    # Initialize a new Trace instance
+    #
+    # @param client [Langsmith::Client] The LangSmith client
+    # @param trace_id [String] ID of the trace
+    def initialize(client, trace_id)
+      @client = client
+      @id = trace_id
+      @runs = []
+      refresh
+    end
+
+    # Refresh the trace data from the API
+    #
+    # @return [Langsmith::Trace] The updated trace
+    def refresh
+      response = @client.get("/traces/#{@id}")
+      @name = response["name"]
+      @start_time = response["start_time"] ? Time.parse(response["start_time"]) : nil
+      @end_time = response["end_time"] ? Time.parse(response["end_time"]) : nil
+      
+      # Get all runs associated with this trace
+      runs_response = @client.get("/runs", { trace_id: @id })
+      @runs = runs_response.map { |run_data| Langsmith::Run.new(@client, run_data) }
+      
+      self
+    end
+
+    # Get the root run of this trace
+    #
+    # @return [Langsmith::Run] The root run
+    def root_run
+      @runs.find { |run| run.parent_run_id.nil? }
+    end
+
+    # Get child runs of a specific run
+    #
+    # @param parent_run_id [String] ID of the parent run
+    # @return [Array<Langsmith::Run>] Child runs
+    def child_runs(parent_run_id)
+      @runs.select { |run| run.parent_run_id == parent_run_id }
+    end
+
+    # Check if the trace is completed
+    #
+    # @return [Boolean] True if the trace is completed
+    def completed?
+      !@end_time.nil?
+    end
+
+    # Get the duration of the trace in seconds
+    #
+    # @return [Float] Duration in seconds, or nil if the trace is not completed
+    def duration
+      return nil unless @end_time && @start_time
+      @end_time - @start_time
+    end
+
+    # Get a hierarchical representation of the trace
+    #
+    # @return [Hash] Hierarchical representation of the trace
+    def to_hierarchy
+      root = root_run
+      return {} unless root
+
+      build_hierarchy(root)
+    end
+
+    private
+
+    def build_hierarchy(run)
+      children = child_runs(run.id)
+      
+      result = {
+        id: run.id,
+        name: run.name,
+        run_type: run.run_type,
+        start_time: run.start_time,
+        end_time: run.end_time,
+        status: run.status,
+        inputs: run.inputs,
+        outputs: run.outputs,
+        error: run.error
+      }
+      
+      result[:children] = children.map { |child| build_hierarchy(child) } unless children.empty?
+      
+      result
+    end
+  end
+end

data/lib/langsmith/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Langsmith
+  VERSION = "0.1.0"
+  Version = VERSION
+end

data/lib/langsmith.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require "logger"
+require "langsmith/version"
+
+module Langsmith
+  class << self
+    # @return [Logger]
+    attr_accessor :logger
+    # @return [Pathname]
+    attr_reader :root
+    # @return [String]
+    attr_accessor :api_key
+    # @return [String]
+    attr_accessor :api_url
+  end
+
+  module Errors
+    class BaseError < StandardError; end
+    class AuthenticationError < BaseError; end
+    class APIError < BaseError; end
+    class ResourceNotFoundError < BaseError; end
+  end
+
+  module Colorizer
+    class << self
+      def red(str)
+        "\e[31m#{str}\e[0m"
+      end
+
+      def green(str)
+        "\e[32m#{str}\e[0m"
+      end
+
+      def yellow(str)
+        "\e[33m#{str}\e[0m"
+      end
+
+      def blue(str)
+        "\e[34m#{str}\e[0m"
+      end
+
+      def colorize_logger_msg(msg, severity)
+        return msg unless msg.is_a?(String)
+
+        return red(msg) if severity.to_sym == :ERROR
+        return yellow(msg) if severity.to_sym == :WARN
+        msg
+      end
+    end
+  end
+
+  LOGGER_OPTIONS = {
+    progname: "Langsmith.rb",
+
+    formatter: ->(severity, time, progname, msg) do
+      Logger::Formatter.new.call(
+        severity,
+        time,
+        "[#{progname}]",
+        Colorizer.colorize_logger_msg(msg, severity)
+      )
+    end
+  }.freeze
+
+  # Default API URL for LangSmith
+  DEFAULT_API_URL = "https://api.smith.langchain.com".freeze
+
+  # Set default logger
+  self.logger ||= ::Logger.new($stdout, **LOGGER_OPTIONS)
+  
+  # Set root path
+  @root = Pathname.new(__dir__)
+  
+  # Set default API URL
+  @api_url = DEFAULT_API_URL
+end
+
+# Load the Langsmith components
+require "langsmith/client"
+require "langsmith/run"
+require "langsmith/dataset"
+require "langsmith/evaluation"
+require "langsmith/feedback"
+require "langsmith/project"
+require "langsmith/trace"

data/lib/langsmithrb.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+# This file is an alias for langsmith.rb to make requiring the gem easier
+require "langsmith"