langsmithrb_rails 0.1.0 → 0.1.1

data/lib/generators/langsmithrb_rails/evals/templates/langsmith_evals.rake

@@ -0,0 +1,255 @@
+# frozen_string_literal: true
+
+require "yaml"
+require "json"
+require "fileutils"
+
+namespace :langsmith do
+  desc "Run an evaluation on a dataset"
+  task :eval, [:dataset, :target, :experiment_name] => :environment do |_t, args|
+    dataset_name = args[:dataset] || "sample"
+    target_name = args[:target] || "http"
+    experiment_name = args[:experiment_name] || "eval_#{Time.now.strftime('%Y%m%d%H%M%S')}"
+    
+    puts "Running evaluation with dataset '#{dataset_name}', target '#{target_name}', experiment '#{experiment_name}'"
+    
+    # Load the dataset
+    dataset_path = Rails.root.join("config/langsmith/evals/datasets/#{dataset_name}.yml")
+    unless File.exist?(dataset_path)
+      puts "Error: Dataset not found at #{dataset_path}"
+      exit 1
+    end
+    
+    dataset = YAML.load_file(dataset_path)
+    puts "Loaded dataset '#{dataset['name']}' with #{dataset['items'].size} items"
+    
+    # Load the target
+    target_path = Rails.root.join("config/langsmith/evals/targets/#{target_name}.rb")
+    unless File.exist?(target_path)
+      puts "Error: Target not found at #{target_path}"
+      exit 1
+    end
+    
+    # Load all check files
+    check_dir = Rails.root.join("config/langsmith/evals/checks")
+    Dir.glob(File.join(check_dir, "*.rb")).each do |file|
+      require file
+    end
+    
+    # Initialize results
+    results = {
+      dataset: dataset["name"],
+      target: target_name,
+      experiment: experiment_name,
+      timestamp: Time.now.iso8601,
+      items: [],
+      summary: {
+        total: dataset["items"].size,
+        passed: 0,
+        failed: 0,
+        avg_score: 0.0
+      }
+    }
+    
+    # Create target instance
+    target_class_name = target_name.capitalize
+    target_class = LangsmithrbRails::Evals::Targets.const_get(target_class_name)
+    target = target_class.new(ENV["LANGSMITH_EVAL_TARGET_CONFIG"] ? JSON.parse(ENV["LANGSMITH_EVAL_TARGET_CONFIG"]) : {})
+    
+    # Run evaluations
+    dataset["items"].each_with_index do |item, index|
+      puts "Evaluating item #{index + 1}/#{dataset['items'].size}: #{item['id']}"
+      
+      # Run the target
+      start_time = Time.now
+      response = target.run(item["input"])
+      end_time = Time.now
+      
+      # Run checks
+      check_results = run_checks(item["input"], response, item["expected_output"])
+      
+      # Determine if passed
+      passed = check_results.values.any? { |r| r[:passed] }
+      
+      # Add to results
+      results[:items] << {
+        id: item["id"],
+        input: item["input"],
+        expected: item["expected_output"],
+        response: response,
+        checks: check_results,
+        passed: passed,
+        duration_ms: ((end_time - start_time) * 1000).to_i,
+        metadata: item["metadata"] || {}
+      }
+      
+      # Update summary
+      results[:summary][:passed] += 1 if passed
+      results[:summary][:failed] += 1 unless passed
+    end
+    
+    # Calculate average score
+    total_score = 0.0
+    total_checks = 0
+    
+    results[:items].each do |item|
+      item[:checks].each do |_name, check|
+        total_score += check[:score]
+        total_checks += 1
+      end
+    end
+    
+    results[:summary][:avg_score] = total_checks > 0 ? (total_score / total_checks) : 0.0
+    
+    # Save results
+    output_dir = Rails.root.join("tmp/langsmith")
+    FileUtils.mkdir_p(output_dir)
+    
+    output_path = output_dir.join("last_eval.json")
+    File.write(output_path, JSON.pretty_generate(results))
+    
+    # Print summary
+    puts "\nEvaluation complete!"
+    puts "Dataset: #{dataset['name']}"
+    puts "Target: #{target_name}"
+    puts "Experiment: #{experiment_name}"
+    puts "Results: #{results[:summary][:passed]}/#{results[:summary][:total]} passed (#{(results[:summary][:avg_score] * 100).round(1)}%)"
+    puts "Results saved to #{output_path}"
+    
+    # Exit with error if below threshold
+    threshold = ENV["LANGSMITH_EVAL_THRESHOLD"] ? ENV["LANGSMITH_EVAL_THRESHOLD"].to_f : 0.7
+    if results[:summary][:avg_score] < threshold
+      puts "\nEvaluation failed: score #{(results[:summary][:avg_score] * 100).round(1)}% is below threshold #{(threshold * 100).round(1)}%"
+      exit 1
+    end
+  end
+  
+  desc "Compare two experiments"
+  task :compare, [:exp_a, :exp_b] => :environment do |_t, args|
+    exp_a = args[:exp_a]
+    exp_b = args[:exp_b]
+    
+    unless exp_a && exp_b
+      puts "Error: Both experiment names are required"
+      puts "Usage: rake langsmith:compare[exp_a,exp_b]"
+      exit 1
+    end
+    
+    # Load experiment results
+    output_dir = Rails.root.join("tmp/langsmith")
+    
+    exp_a_path = output_dir.join("#{exp_a}.json")
+    exp_b_path = output_dir.join("#{exp_b}.json")
+    
+    unless File.exist?(exp_a_path) && File.exist?(exp_b_path)
+      puts "Error: Experiment results not found"
+      exit 1
+    end
+    
+    exp_a_results = JSON.parse(File.read(exp_a_path))
+    exp_b_results = JSON.parse(File.read(exp_b_path))
+    
+    # Compare results
+    comparison = {
+      exp_a: {
+        name: exp_a,
+        passed: exp_a_results["summary"]["passed"],
+        total: exp_a_results["summary"]["total"],
+        avg_score: exp_a_results["summary"]["avg_score"]
+      },
+      exp_b: {
+        name: exp_b,
+        passed: exp_b_results["summary"]["passed"],
+        total: exp_b_results["summary"]["total"],
+        avg_score: exp_b_results["summary"]["avg_score"]
+      },
+      diff: {
+        passed: exp_b_results["summary"]["passed"] - exp_a_results["summary"]["passed"],
+        avg_score: exp_b_results["summary"]["avg_score"] - exp_a_results["summary"]["avg_score"]
+      },
+      items: []
+    }
+    
+    # Compare individual items
+    exp_a_items = exp_a_results["items"].index_by { |item| item["id"] }
+    exp_b_items = exp_b_results["items"].index_by { |item| item["id"] }
+    
+    all_item_ids = (exp_a_items.keys + exp_b_items.keys).uniq
+    
+    all_item_ids.each do |id|
+      item_a = exp_a_items[id]
+      item_b = exp_b_items[id]
+      
+      next unless item_a && item_b
+      
+      # Calculate score difference
+      score_a = calculate_avg_score(item_a["checks"])
+      score_b = calculate_avg_score(item_b["checks"])
+      
+      comparison[:items] << {
+        id: id,
+        exp_a: {
+          passed: item_a["passed"],
+          score: score_a
+        },
+        exp_b: {
+          passed: item_b["passed"],
+          score: score_b
+        },
+        diff: {
+          passed: item_b["passed"] != item_a["passed"],
+          score: score_b - score_a
+        }
+      }
+    end
+    
+    # Sort by score difference
+    comparison[:items].sort_by! { |item| -item[:diff][:score].abs }
+    
+    # Save comparison
+    comparison_path = output_dir.join("comparison_#{exp_a}_#{exp_b}.json")
+    File.write(comparison_path, JSON.pretty_generate(comparison))
+    
+    # Print summary
+    puts "\nComparison complete!"
+    puts "Experiment A: #{exp_a} - #{comparison[:exp_a][:passed]}/#{comparison[:exp_a][:total]} passed (#{(comparison[:exp_a][:avg_score] * 100).round(1)}%)"
+    puts "Experiment B: #{exp_b} - #{comparison[:exp_b][:passed]}/#{comparison[:exp_b][:total]} passed (#{(comparison[:exp_b][:avg_score] * 100).round(1)}%)"
+    puts "Difference: #{comparison[:diff][:passed] >= 0 ? '+' : ''}#{comparison[:diff][:passed]} passed, #{comparison[:diff][:avg_score] >= 0 ? '+' : ''}#{(comparison[:diff][:avg_score] * 100).round(1)}%"
+    puts "Results saved to #{comparison_path}"
+    
+    # Print top differences
+    puts "\nTop differences:"
+    comparison[:items].first(5).each do |item|
+      puts "#{item[:id]}: #{(item[:diff][:score] * 100).round(1)}% #{item[:diff][:score] >= 0 ? 'improvement' : 'regression'}"
+    end
+  end
+  
+  private
+  
+  # Run all available checks on a response
+  def run_checks(input, response, expected)
+    results = {}
+    
+    # Find all check classes
+    LangsmithrbRails::Evals::Checks.constants.each do |const|
+      check_class = LangsmithrbRails::Evals::Checks.const_get(const)
+      next unless check_class.respond_to?(:evaluate)
+      
+      # Run the check
+      check_name = const.to_s.underscore
+      results[check_name] = check_class.evaluate(input, response, expected)
+    end
+    
+    results
+  end
+  
+  # Calculate average score for an item
+  def calculate_avg_score(checks)
+    total_score = 0.0
+    checks.each do |_name, check|
+      total_score += check["score"]
+    end
+    
+    checks.empty? ? 0.0 : (total_score / checks.size)
+  end
+end

data/lib/generators/langsmithrb_rails/evals/templates/targets/http.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "uri"
+require "json"
+
+module LangsmithrbRails
+  module Evals
+    module Targets
+      # HTTP target for evaluating LLM responses via API calls
+      class Http
+        # Initialize the target
+        # @param config [Hash] Configuration options
+        def initialize(config = {})
+          @url = config["url"] || ENV["LANGSMITH_EVAL_TARGET_URL"]
+          @headers = config["headers"] || {}
+          @method = (config["method"] || "post").downcase
+          @timeout = config["timeout"] || 30
+          
+          # Add default headers
+          @headers["Content-Type"] ||= "application/json"
+          @headers["Accept"] ||= "application/json"
+          
+          # Add authorization if provided
+          if config["api_key"]
+            @headers["Authorization"] ||= "Bearer #{config["api_key"]}"
+          end
+        end
+        
+        # Run the target with the given input
+        # @param input [Hash] Input data
+        # @return [Hash] Response data
+        def run(input)
+          # Validate URL
+          unless @url
+            return { error: "No target URL specified" }
+          end
+          
+          # Create URI
+          uri = URI.parse(@url)
+          
+          # Create HTTP client
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = uri.scheme == "https"
+          http.open_timeout = @timeout
+          http.read_timeout = @timeout
+          
+          # Create request
+          request = create_request(uri, input)
+          
+          # Add headers
+          @headers.each do |key, value|
+            request[key] = value
+          end
+          
+          # Send request
+          response = http.request(request)
+          
+          # Parse response
+          parse_response(response)
+        rescue => e
+          { error: e.message }
+        end
+        
+        private
+        
+        # Create the HTTP request
+        # @param uri [URI] Request URI
+        # @param input [Hash] Input data
+        # @return [Net::HTTPRequest] HTTP request
+        def create_request(uri, input)
+          case @method
+          when "get"
+            # For GET requests, add parameters to the URL
+            query = URI.encode_www_form(input)
+            uri.query = query
+            Net::HTTP::Get.new(uri)
+          when "post"
+            # For POST requests, add parameters to the body
+            request = Net::HTTP::Post.new(uri)
+            request.body = JSON.generate(input)
+            request
+          when "put"
+            # For PUT requests, add parameters to the body
+            request = Net::HTTP::Put.new(uri)
+            request.body = JSON.generate(input)
+            request
+          else
+            # Default to POST
+            request = Net::HTTP::Post.new(uri)
+            request.body = JSON.generate(input)
+            request
+          end
+        end
+        
+        # Parse the HTTP response
+        # @param response [Net::HTTPResponse] HTTP response
+        # @return [Hash] Parsed response
+        def parse_response(response)
+          if response.code.to_i >= 200 && response.code.to_i < 300
+            # Successful response
+            if response["Content-Type"]&.include?("application/json")
+              # Parse JSON response
+              JSON.parse(response.body)
+            else
+              # Return raw response
+              { text: response.body }
+            end
+          else
+            # Error response
+            {
+              error: "HTTP error #{response.code}",
+              body: response.body
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/generators/langsmithrb_rails/evals/templates/targets/ruby.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+module LangsmithrbRails
+  module Evals
+    module Targets
+      # Ruby target for evaluating LLM responses via direct Ruby calls
+      class Ruby
+        # Initialize the target
+        # @param config [Hash] Configuration options
+        def initialize(config = {})
+          @class_name = config["class"]
+          @method_name = config["method"]
+          @args = config["args"] || []
+          @kwargs = config["kwargs"] || {}
+        end
+        
+        # Run the target with the given input
+        # @param input [Hash] Input data
+        # @return [Hash] Response data
+        def run(input)
+          # Validate configuration
+          unless @class_name && @method_name
+            return { error: "Class and method must be specified" }
+          end
+          
+          # Find the class
+          klass = find_class(@class_name)
+          unless klass
+            return { error: "Class #{@class_name} not found" }
+          end
+          
+          # Check if the method exists
+          unless klass.respond_to?(@method_name) || klass.instance_methods.include?(@method_name.to_sym)
+            return { error: "Method #{@method_name} not found in #{@class_name}" }
+          end
+          
+          # Prepare arguments
+          args = prepare_args(input)
+          kwargs = prepare_kwargs(input)
+          
+          # Call the method
+          result = call_method(klass, args, kwargs)
+          
+          # Format the result
+          format_result(result)
+        rescue => e
+          { error: e.message, backtrace: e.backtrace&.first(5) }
+        end
+        
+        private
+        
+        # Find the class by name
+        # @param class_name [String] Class name
+        # @return [Class, nil] Class or nil if not found
+        def find_class(class_name)
+          class_name.split("::").inject(Object) do |mod, class_name|
+            mod.const_get(class_name)
+          end
+        rescue NameError
+          nil
+        end
+        
+        # Prepare arguments for the method call
+        # @param input [Hash] Input data
+        # @return [Array] Arguments for the method call
+        def prepare_args(input)
+          @args.map do |arg|
+            if arg.is_a?(String) && arg.start_with?("$")
+              # Replace placeholders with input values
+              key = arg[1..-1]
+              input[key] || input[key.to_sym]
+            else
+              arg
+            end
+          end
+        end
+        
+        # Prepare keyword arguments for the method call
+        # @param input [Hash] Input data
+        # @return [Hash] Keyword arguments for the method call
+        def prepare_kwargs(input)
+          result = {}
+          
+          @kwargs.each do |key, value|
+            if value.is_a?(String) && value.start_with?("$")
+              # Replace placeholders with input values
+              input_key = value[1..-1]
+              result[key.to_sym] = input[input_key] || input[input_key.to_sym]
+            else
+              result[key.to_sym] = value
+            end
+          end
+          
+          # Add all input values as keyword arguments if not already present
+          input.each do |key, value|
+            key_sym = key.to_sym
+            result[key_sym] = value unless result.key?(key_sym)
+          end
+          
+          result
+        end
+        
+        # Call the method
+        # @param klass [Class] Class to call the method on
+        # @param args [Array] Arguments for the method call
+        # @param kwargs [Hash] Keyword arguments for the method call
+        # @return [Object] Result of the method call
+        def call_method(klass, args, kwargs)
+          if klass.respond_to?(@method_name)
+            # Call class method
+            klass.send(@method_name, *args, **kwargs)
+          else
+            # Call instance method
+            klass.new.send(@method_name, *args, **kwargs)
+          end
+        end
+        
+        # Format the result
+        # @param result [Object] Result of the method call
+        # @return [Hash] Formatted result
+        def format_result(result)
+          case result
+          when Hash
+            result
+          when String
+            { text: result }
+          when Array
+            { items: result }
+          else
+            { output: result }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/generators/langsmithrb_rails/install/install_generator.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module LangsmithrbRails
+  module Generators
+    # Generator for installing LangSmith Rails integration
+    class InstallGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+      
+      desc "Creates LangSmith configuration files for your Rails application"
+      
+      def create_initializer
+        template "initializer.rb", "config/initializers/langsmith.rb"
+      end
+      
+      def create_config_file
+        template "config.yml", "config/langsmith.yml"
+      end
+      
+      def display_post_install_message
+        say "\n"
+        say "LangSmith Rails has been installed! 🎉", :green
+        say "\n"
+        say "Next steps:", :yellow
+        say "  1. Add your LangSmith API key to your environment:", :yellow
+        say "     export LANGSMITH_API_KEY=your_api_key", :yellow
+        say "  2. Optionally set your LangSmith project name:", :yellow
+        say "     export LANGSMITH_PROJECT=your_project_name", :yellow
+        say "\n"
+        say "To enable tracing in your Rails app, run:", :yellow
+        say "  bin/rails g langsmithrb_rails:tracing", :yellow
+        say "\n"
+      end
+    end
+  end
+end

data/lib/generators/langsmithrb_rails/install/templates/config.yml

@@ -0,0 +1,45 @@
+# LangSmith Rails Configuration
+# This file was generated by the langsmithrb_rails:install generator.
+# You can customize this file to configure LangSmith for your Rails application.
+
+# Default configuration for all environments
+default: &default
+  # LangSmith API URL
+  api_url: https://api.smith.langchain.com
+  
+  # Whether to redact PII by default
+  redact_by_default: true
+  
+  # HTTP request timeouts (in seconds)
+  timeout_seconds: 3.0
+  open_timeout_seconds: 1.0
+
+# Development environment configuration
+development:
+  <<: *default
+  # Enable tracing in development
+  enabled: true
+  # Sample 100% of requests in development
+  sampling_rate: 1.0
+  # Disable PII redaction in development
+  redact_by_default: false
+
+# Test environment configuration
+test:
+  <<: *default
+  # Enable tracing in test
+  enabled: true
+  # Sample 100% of requests in test
+  sampling_rate: 1.0
+  # Disable PII redaction in test
+  redact_by_default: false
+
+# Production environment configuration
+production:
+  <<: *default
+  # Enable tracing in production if API key is present
+  enabled: <%= ENV["LANGSMITH_API_KEY"].present? %>
+  # Sample 10% of requests in production
+  sampling_rate: 0.1
+  # Enable PII redaction in production
+  redact_by_default: true

data/lib/generators/langsmithrb_rails/install/templates/initializer.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# LangSmith Rails Initializer
+# This file was generated by the langsmithrb_rails:install generator.
+# You can customize this file to configure LangSmith for your Rails application.
+
+# Configure LangSmith Rails
+LangsmithrbRails.configure do |config|
+  # Whether LangSmith tracing is enabled
+  # Defaults to true if LANGSMITH_API_KEY is present
+  config.enabled = ENV["LANGSMITH_API_KEY"].present?
+  
+  # LangSmith API key
+  # Set this to your LangSmith API key
+  config.api_key = ENV["LANGSMITH_API_KEY"]
+  
+  # LangSmith project name
+  # Set this to your LangSmith project name
+  config.project_name = ENV["LANGSMITH_PROJECT"]
+end
+
+# Configure LangSmith client directly if enabled
+if LangsmithrbRails.config.enabled && LangsmithrbRails.config.api_key
+  Langsmithrb.configure do |config|
+    config.api_key = LangsmithrbRails.config.api_key
+    config.project_name = LangsmithrbRails.config.project_name
+    config.tracing_enabled = true
+  end
+end
+
+# Load environment-specific configuration from config/langsmith.yml
+Rails.application.config.after_initialize do
+  LangsmithrbRails::Config.load!(rails_root: Rails.root)
+end

data/lib/generators/langsmithrb_rails/privacy/privacy_generator.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module LangsmithrbRails
+  module Generators
+    # Generator for adding privacy features to Rails applications
+    class PrivacyGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+      
+      desc "Adds privacy features for LangSmith traces"
+      
+      def create_custom_redactor
+        template "custom_redactor.rb", "app/lib/langsmithrb_rails/custom_redactor.rb"
+      end
+      
+      def create_initializer
+        template "privacy_initializer.rb", "config/initializers/langsmith_privacy.rb"
+      end
+      
+      def create_config_file
+        template "privacy.yml", "config/langsmith_privacy.yml"
+      end
+      
+      def display_post_install_message
+        say "\n"
+        say "LangSmith privacy features have been added to your Rails application! 🎉", :green
+        say "\n"
+        say "This adds:", :yellow
+        say "  1. Custom redactor for PII in app/lib/langsmithrb_rails/custom_redactor.rb", :yellow
+        say "  2. Privacy initializer in config/initializers/langsmith_privacy.rb", :yellow
+        say "  3. Privacy configuration in config/langsmith_privacy.yml", :yellow
+        say "\n"
+        say "To customize redaction:", :yellow
+        say "  1. Edit config/langsmith_privacy.yml to configure allowlists and patterns", :yellow
+        say "  2. Modify app/lib/langsmithrb_rails/custom_redactor.rb to add custom patterns", :yellow
+        say "\n"
+      end
+    end
+  end
+end