langsmithrb_rails 0.1.1 → 0.3.0

data/lib/langsmithrb_rails/client.rb

@@ -3,6 +3,7 @@
 require "net/http"
 require "json"
 require "uri"
+require "securerandom"
 
 module LangsmithrbRails
   # Direct REST client for LangSmith API
@@ -30,8 +31,196 @@ module LangsmithrbRails
       patch("/runs/#{id}", payload)
     end
 
+    # Get a run by ID
+    # @param id [String] Run ID
+    # @return [Hash] Response with status and body
+    def get_run(id)
+      get("/runs/#{id}")
+    end
+
+    # List runs with optional filters
+    # @param project_name [String] Filter by project name
+    # @param trace_id [String] Filter by trace ID
+    # @param run_type [String] Filter by run type
+    # @param limit [Integer] Maximum number of runs to return
+    # @param offset [Integer] Offset for pagination
+    # @return [Hash] Response with status and body
+    def list_runs(project_name: nil, trace_id: nil, run_type: nil, limit: 100, offset: 0)
+      query_params = {
+        project_name: project_name,
+        trace_id: trace_id,
+        run_type: run_type,
+        limit: limit,
+        offset: offset
+      }.compact
+      
+      query_string = query_params.map { |k, v| "#{k}=#{URI.encode_www_form_component(v.to_s)}" }.join("&")
+      get("/runs?#{query_string}")
+    end
+
+    # Create a dataset in LangSmith
+    # @param name [String] Dataset name
+    # @param description [String] Dataset description
+    # @return [Hash] Response with status and body
+    def create_dataset(name, description: nil)
+      payload = {
+        name: name,
+        description: description
+      }.compact
+      post("/datasets", payload)
+    end
+
+    # Get a dataset by ID
+    # @param id [String] Dataset ID
+    # @return [Hash] Response with status and body
+    def get_dataset(id)
+      get("/datasets/#{id}")
+    end
+
+    # List datasets
+    # @param limit [Integer] Maximum number of datasets to return
+    # @param offset [Integer] Offset for pagination
+    # @return [Hash] Response with status and body
+    def list_datasets(limit: 100, offset: 0)
+      get("/datasets?limit=#{limit}&offset=#{offset}")
+    end
+
+    # Create an example in a dataset
+    # @param dataset_id [String] Dataset ID
+    # @param inputs [Hash] Example inputs
+    # @param outputs [Hash] Example outputs (optional)
+    # @return [Hash] Response with status and body
+    def create_example(dataset_id, inputs, outputs = nil)
+      payload = {
+        dataset_id: dataset_id,
+        inputs: inputs,
+        outputs: outputs
+      }.compact
+      post("/examples", payload)
+    end
+
+    # Get an example by ID
+    # @param id [String] Example ID
+    # @return [Hash] Response with status and body
+    def get_example(id)
+      get("/examples/#{id}")
+    end
+
+    # List examples in a dataset
+    # @param dataset_id [String] Dataset ID
+    # @param limit [Integer] Maximum number of examples to return
+    # @param offset [Integer] Offset for pagination
+    # @return [Hash] Response with status and body
+    def list_examples(dataset_id, limit: 100, offset: 0)
+      get("/datasets/#{dataset_id}/examples?limit=#{limit}&offset=#{offset}")
+    end
+
+    # Create feedback for a run
+    # @param run_id [String] Run ID
+    # @param key [String] Feedback key
+    # @param score [Float, Integer, Boolean] Feedback score
+    # @param value [Hash] Additional feedback data (optional)
+    # @param comment [String] Feedback comment (optional)
+    # @return [Hash] Response with status and body
+    def create_feedback(run_id, key, score, value: nil, comment: nil)
+      payload = {
+        id: SecureRandom.uuid,
+        run_id: run_id,
+        key: key,
+        score: score,
+        value: value,
+        comment: comment
+      }.compact
+      post("/feedback", payload)
+    end
+
+    # Get feedback by ID
+    # @param id [String] Feedback ID
+    # @return [Hash] Response with status and body
+    def get_feedback(id)
+      get("/feedback/#{id}")
+    end
+
+    # List feedback for a run
+    # @param run_id [String] Run ID
+    # @return [Hash] Response with status and body
+    def list_feedback(run_id)
+      get("/runs/#{run_id}/feedback")
+    end
+
+    # Create a project in LangSmith
+    # @param name [String] Project name
+    # @param description [String] Project description (optional)
+    # @return [Hash] Response with status and body
+    def create_project(name, description: nil)
+      payload = {
+        name: name,
+        description: description
+      }.compact
+      post("/projects", payload)
+    end
+
+    # Get a project by name
+    # @param name [String] Project name
+    # @return [Hash] Response with status and body
+    def get_project(name)
+      get("/projects/#{URI.encode_www_form_component(name)}")
+    end
+
+    # List projects
+    # @param limit [Integer] Maximum number of projects to return
+    # @param offset [Integer] Offset for pagination
+    # @return [Hash] Response with status and body
+    def list_projects(limit: 100, offset: 0)
+      get("/projects?limit=#{limit}&offset=#{offset}")
+    end
+
+    # Upload a file attachment
+    # @param file_path [String] Path to the file
+    # @return [Hash] Response with status and body containing the file ID
+    def upload_file(file_path)
+      uri = URI.parse("#{@api_url}/files")
+      
+      File.open(file_path, 'rb') do |file|
+        boundary = SecureRandom.hex(16)
+        
+        req = Net::HTTP::Post.new(uri.request_uri)
+        req['Authorization'] = "Bearer #{@api_key}" if @api_key
+        req['Content-Type'] = "multipart/form-data; boundary=#{boundary}"
+        
+        body = []
+        body << "--#{boundary}\r\n"
+        body << "Content-Disposition: form-data; name=\"file\"; filename=\"#{File.basename(file_path)}\"\r\n"
+        body << "Content-Type: #{content_type_for_file(file_path)}\r\n\r\n"
+        body << file.read
+        body << "\r\n--#{boundary}--\r\n"
+        
+        req.body = body.join
+        
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = Config[:open_timeout_seconds]
+        http.read_timeout = Config[:timeout_seconds]
+        
+        res = http.request(req)
+        { 
+          status: res.code.to_i, 
+          body: (JSON.parse(res.body) rescue { "raw" => res.body })
+        }
+      end
+    rescue => e
+      { status: 0, error: e.message }
+    end
+
     private
 
+    # Make a GET request
+    # @param path [String] API path
+    # @return [Hash] Response with status and body
+    def get(path)
+      request(Net::HTTP::Get, path, nil)
+    end
+
     # Make a POST request
     # @param path [String] API path
     # @param payload [Hash] Request payload
@@ -48,6 +237,13 @@ module LangsmithrbRails
       request(Net::HTTP::Patch, path, payload)
     end
 
+    # Make a DELETE request
+    # @param path [String] API path
+    # @return [Hash] Response with status and body
+    def delete(path)
+      request(Net::HTTP::Delete, path, nil)
+    end
+
     # Make an HTTP request
     # @param klass [Class] Net::HTTP request class
     # @param path [String] API path
@@ -62,8 +258,11 @@ module LangsmithrbRails
       
       req = klass.new(uri.request_uri)
       req["Authorization"] = "Bearer #{@api_key}" if @api_key
-      req["Content-Type"] = "application/json"
-      req.body = JSON.generate(payload)
+      
+      if payload
+        req["Content-Type"] = "application/json"
+        req.body = JSON.generate(payload)
+      end
       
       res = http.request(req)
       { 
@@ -73,5 +272,21 @@ module LangsmithrbRails
     rescue => e
       { status: 0, error: e.message }
     end
+
+    # Determine content type for a file
+    # @param file_path [String] Path to the file
+    # @return [String] Content type
+    def content_type_for_file(file_path)
+      ext = File.extname(file_path).downcase
+      case ext
+      when ".jpg", ".jpeg" then "image/jpeg"
+      when ".png" then "image/png"
+      when ".pdf" then "application/pdf"
+      when ".txt" then "text/plain"
+      when ".json" then "application/json"
+      when ".csv" then "text/csv"
+      else "application/octet-stream"
+      end
+    end
   end
 end

data/lib/langsmithrb_rails/config.rb

@@ -2,71 +2,168 @@
 
 require "yaml"
 require "ostruct"
+require "logger"
+require "singleton"
 
 module LangsmithrbRails
   # Configuration class for LangsmithrbRails
   class Config
+    include Singleton
+    
     # Default configuration values
     DEFAULTS = {
       api_url: "https://api.smith.langchain.com",
-      project: nil,
+      project_name: "default",
       api_key: ENV["LANGSMITH_API_KEY"],
-      sampling_rate: ENV.fetch("LANGSMITH_SAMPLING", "0.1").to_f,
+      sampling: 1.0,
       redact_by_default: true,
       timeout_seconds: 3.0,
       open_timeout_seconds: 1.0,
       env: ENV["RAILS_ENV"] || "development",
-      enabled: false
+      enabled: true,
+      # Advanced tracing options
+      trace_all: ENV.fetch("LANGSMITH_TRACE_ALL", "false") == "true",
+      trace_level: ENV.fetch("LANGSMITH_TRACE_LEVEL", "info").to_sym,
+      # OpenTelemetry options
+      otel_enabled: ENV.fetch("LANGSMITH_OTEL_ENABLED", "false") == "true",
+      otel_service_name: ENV.fetch("LANGSMITH_OTEL_SERVICE_NAME", "langsmithrb_rails"),
+      # Evaluation options
+      evaluation_enabled: ENV.fetch("LANGSMITH_EVALUATION_ENABLED", "false") == "true",
+      # Logging options
+      log_level: ENV.fetch("LANGSMITH_LOG_LEVEL", "info").to_sym,
+      log_to_stdout: ENV.fetch("LANGSMITH_LOG_TO_STDOUT", "false") == "true"
     }.freeze
 
-    class << self
-      # Load configuration from YAML file and environment variables
-      # @param rails_root [Pathname] Rails root path
-      # @return [OpenStruct] Configuration object
-      def load!(rails_root: Rails.root)
-        file = rails_root.join("config/langsmith.yml")
-        yml = File.exist?(file) ? YAML.load_file(file) : {}
-        env = (ENV["RAILS_ENV"] || "development").to_s
-        @config = OpenStruct.new(DEFAULTS.merge(yml.fetch(env, {})))
-        
-        # Override with environment variables if present
-        @config.api_key = ENV["LANGSMITH_API_KEY"] if ENV["LANGSMITH_API_KEY"]
-        @config.project = ENV["LANGSMITH_PROJECT"] if ENV["LANGSMITH_PROJECT"]
-        @config.enabled = @config.api_key.present? unless ENV.key?("LANGSMITH_ENABLED")
-        @config.enabled = ENV["LANGSMITH_ENABLED"] == "true" if ENV.key?("LANGSMITH_ENABLED")
-        
-        @config
-      end
+    # Configuration attributes
+    attr_accessor :api_key, :api_url, :project_name, :enabled, :sampling,
+                  :trace_all, :trace_level, :otel_enabled, :otel_service_name,
+                  :evaluation_enabled, :log_level, :log_to_stdout
+    attr_reader :logger
 
-      # Access configuration values
-      # @param key [Symbol] Configuration key
-      # @return [Object] Configuration value
-      def [](key)
-        @config&.send(key)
+    # Initialize with default values
+    def initialize
+      # Set defaults first
+      DEFAULTS.each do |key, value|
+        instance_variable_set("@#{key}", value)
       end
-
-      # Set configuration values
-      # @param key [Symbol] Configuration key
-      # @param value [Object] Configuration value
-      # @return [Object] Configuration value
-      def set(key, value)
-        @config&.send("#{key}=", value)
+      
+      # Then override with environment variables if present
+      load_from_env
+      
+      # Initialize logger
+      @logger = create_logger(@log_level, @log_to_stdout)
+    end
+    
+    # Load configuration from environment variables
+    # @return [Config] Self for chaining
+    def load_from_env
+      @api_key = ENV["LANGSMITH_API_KEY"] if ENV["LANGSMITH_API_KEY"]
+      @api_url = ENV["LANGSMITH_API_URL"] if ENV["LANGSMITH_API_URL"]
+      @project_name = ENV["LANGSMITH_PROJECT"] if ENV["LANGSMITH_PROJECT"]
+      @enabled = ENV["LANGSMITH_ENABLED"] == "true" if ENV.key?("LANGSMITH_ENABLED")
+      @sampling = ENV["LANGSMITH_SAMPLING"].to_f if ENV["LANGSMITH_SAMPLING"]
+      @trace_all = ENV["LANGSMITH_TRACE_ALL"] == "true" if ENV.key?("LANGSMITH_TRACE_ALL")
+      @trace_level = ENV["LANGSMITH_TRACE_LEVEL"].to_sym if ENV["LANGSMITH_TRACE_LEVEL"]
+      @otel_enabled = ENV["LANGSMITH_OTEL_ENABLED"] == "true" if ENV.key?("LANGSMITH_OTEL_ENABLED")
+      @otel_service_name = ENV["LANGSMITH_OTEL_SERVICE_NAME"] if ENV["LANGSMITH_OTEL_SERVICE_NAME"]
+      @evaluation_enabled = ENV["LANGSMITH_EVALUATION_ENABLED"] == "true" if ENV.key?("LANGSMITH_EVALUATION_ENABLED")
+      @log_level = ENV["LANGSMITH_LOG_LEVEL"].to_sym if ENV["LANGSMITH_LOG_LEVEL"]
+      @log_to_stdout = ENV["LANGSMITH_LOG_TO_STDOUT"] == "true" if ENV.key?("LANGSMITH_LOG_TO_STDOUT")
+      self
+    end
+    
+    # Load configuration from YAML file
+    # @param path [String] Path to YAML file
+    # @return [Config] Self for chaining
+    def load_from_yaml(path)
+      return self unless File.exist?(path)
+      
+      yml = YAML.load_file(path)
+      env = (ENV["RAILS_ENV"] || "development").to_s
+      config = yml.fetch(env, {})
+      
+      config.each do |key, value|
+        send("#{key}=", value) if respond_to?("#{key}=")
       end
-
-      # Get current configuration
-      # @return [OpenStruct] Configuration object
-      def current
-        @config ||= OpenStruct.new(DEFAULTS)
+      
+      # Reset logger if log settings changed
+      @logger = create_logger(@log_level, @log_to_stdout)
+      
+      self
+    end
+    
+    # Check if LangSmith is enabled
+    # @return [Boolean] Whether LangSmith is enabled
+    def enabled?
+      @enabled
+    end
+    
+    # Check if a trace should be sampled
+    # @return [Boolean] Whether to sample the trace
+    def should_sample?
+      return true if @sampling >= 1.0
+      return false if @sampling <= 0.0
+      Random.rand < @sampling
+    end
+    
+    # Check if OpenTelemetry is enabled
+    # @return [Boolean] Whether OpenTelemetry is enabled
+    def otel_enabled?
+      @otel_enabled
+    end
+    
+    # Check if evaluation is enabled
+    # @return [Boolean] Whether evaluation is enabled
+    def evaluation_enabled?
+      @evaluation_enabled
+    end
+    
+    # Set log level
+    # @param level [Symbol, String] Log level
+    # @return [Symbol] Log level
+    def log_level=(level)
+      level = level.to_sym if level.is_a?(String)
+      unless [:debug, :info, :warn, :error, :fatal].include?(level)
+        raise ArgumentError, "Invalid log level: #{level}"
       end
+      @log_level = level
+      @logger = create_logger(@log_level, @log_to_stdout)
+      @log_level
     end
-
-    # For backwards compatibility
-    attr_accessor :enabled, :api_key, :project_name
-
-    def initialize
-      @enabled = false
-      @api_key = nil
-      @project_name = nil
+    
+    # Get logger instance
+    # @return [Logger] Logger instance
+    def logger
+      @logger ||= create_logger(@log_level, @log_to_stdout)
+    end
+    
+    private
+    
+    # Create a logger
+    # @param level [Symbol] Log level
+    # @param to_stdout [Boolean] Whether to log to STDOUT
+    # @return [Logger] Logger instance
+    def create_logger(level, to_stdout)
+      logger = Logger.new(to_stdout ? STDOUT : IO::NULL)
+      logger.level = parse_log_level(level)
+      logger.formatter = proc do |severity, datetime, progname, msg|
+        "[LangsmithrbRails] [#{severity}] #{msg}\n"
+      end
+      logger
+    end
+    
+    # Parse log level
+    # @param level [Symbol, String] Log level
+    # @return [Integer] Logger level constant
+    def parse_log_level(level)
+      case level.to_s.downcase
+      when "debug" then Logger::DEBUG
+      when "info" then Logger::INFO
+      when "warn" then Logger::WARN
+      when "error" then Logger::ERROR
+      when "fatal" then Logger::FATAL
+      else Logger::INFO
+      end
     end
   end
 end

data/lib/langsmithrb_rails/evaluation/evaluator.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+module LangsmithrbRails
+  module Evaluation
+    # Base evaluator class
+    class Evaluator
+      attr_reader :client, :project_name, :tags
+
+      # Initialize a new evaluator
+      # @param client [LangsmithrbRails::Client] LangSmith client
+      # @param project_name [String] Optional project name for evaluations
+      # @param tags [Array<String>] Optional tags for evaluations
+      def initialize(client: nil, project_name: nil, tags: [])
+        @client = client || LangsmithrbRails::Client.new
+        @project_name = project_name
+        @tags = tags
+      end
+
+      # Evaluate a prediction against a reference
+      # @param prediction [String, Hash] The prediction to evaluate
+      # @param reference [String, Hash] The reference to compare against
+      # @param input [Hash] Optional input that generated the prediction
+      # @return [Hash] Evaluation result with score and metadata
+      def evaluate(prediction, reference = nil, input = nil)
+        raise NotImplementedError, "Subclasses must implement evaluate method"
+      end
+
+      # Evaluate a run
+      # @param run_id [String] The ID of the run to evaluate
+      # @param reference [String, Hash] The reference to compare against
+      # @return [Hash] Evaluation result with score and metadata
+      def evaluate_run(run_id, reference = nil)
+        # Get the run
+        response = client.get_run(run_id)
+        
+        unless response[:status] >= 200 && response[:status] < 300
+          raise "Failed to get run: #{response[:error] || response[:body]}"
+        end
+        
+        run = response[:body]
+        
+        # Extract prediction from run outputs
+        prediction = extract_prediction_from_run(run)
+        
+        # Extract input from run inputs
+        input = run["inputs"]
+        
+        # Evaluate
+        result = evaluate(prediction, reference, input)
+        
+        # Create feedback
+        create_feedback(run_id, result)
+        
+        result
+      end
+
+      # Evaluate multiple runs
+      # @param run_ids [Array<String>] The IDs of the runs to evaluate
+      # @param references [Hash<String, Object>] Map of run IDs to references
+      # @return [Hash<String, Hash>] Map of run IDs to evaluation results
+      def evaluate_runs(run_ids, references = {})
+        results = {}
+        
+        run_ids.each do |run_id|
+          reference = references[run_id]
+          results[run_id] = evaluate_run(run_id, reference)
+        end
+        
+        results
+      end
+
+      # Evaluate a dataset
+      # @param dataset_id [String] The ID of the dataset to evaluate
+      # @param experiment_name [String] Name for the experiment
+      # @param target_llm [Object] Optional LLM to use for generating predictions
+      # @return [Hash] Evaluation results for the dataset
+      def evaluate_dataset(dataset_id, experiment_name, target_llm = nil)
+        # Get the dataset examples
+        response = client.list_examples(dataset_id)
+        
+        unless response[:status] >= 200 && response[:status] < 300
+          raise "Failed to get dataset examples: #{response[:error] || response[:body]}"
+        end
+        
+        examples = response[:body]
+        
+        results = {
+          experiment_name: experiment_name,
+          dataset_id: dataset_id,
+          evaluator_name: self.class.name,
+          results: []
+        }
+        
+        examples.each do |example|
+          # If target LLM is provided, generate a prediction
+          if target_llm
+            prediction = generate_prediction(target_llm, example["inputs"])
+          else
+            # Otherwise use the example's outputs as the prediction
+            prediction = example["outputs"]
+          end
+          
+          # Evaluate
+          result = evaluate(prediction, example["outputs"], example["inputs"])
+          
+          results[:results] << {
+            example_id: example["id"],
+            score: result[:score],
+            metadata: result[:metadata]
+          }
+        end
+        
+        results
+      end
+
+      private
+
+      # Extract prediction from run outputs
+      # @param run [Hash] The run data
+      # @return [Object] The prediction
+      def extract_prediction_from_run(run)
+        outputs = run["outputs"] || {}
+        
+        # Try common output keys
+        %w[output response result text completion answer].each do |key|
+          return outputs[key] if outputs.key?(key)
+        end
+        
+        # If no common keys found, return the entire outputs
+        outputs
+      end
+
+      # Generate a prediction using an LLM
+      # @param llm [Object] The LLM to use
+      # @param input [Hash] The input to generate from
+      # @return [Object] The generated prediction
+      def generate_prediction(llm, input)
+        if llm.respond_to?(:call)
+          llm.call(input)
+        elsif llm.respond_to?(:generate)
+          llm.generate(input)
+        elsif llm.respond_to?(:complete)
+          llm.complete(input)
+        else
+          raise "Unsupported LLM interface"
+        end
+      end
+
+      # Create feedback for a run
+      # @param run_id [String] The ID of the run
+      # @param result [Hash] The evaluation result
+      # @return [Hash] The created feedback
+      def create_feedback(run_id, result)
+        feedback_data = {
+          run_id: run_id,
+          key: self.class.name.split("::").last.downcase,
+          score: result[:score],
+          value: result[:metadata],
+          comment: result[:comment]
+        }.compact
+        
+        response = client.create_feedback(
+          feedback_data[:run_id],
+          feedback_data[:key],
+          feedback_data[:score],
+          value: feedback_data[:value],
+          comment: feedback_data[:comment]
+        )
+        
+        unless response[:status] >= 200 && response[:status] < 300
+          LangsmithrbRails.logger.error("Failed to create feedback: #{response[:error] || response[:body]}")
+        end
+        
+        response[:body]
+      end
+    end
+  end
+end