langsmithrb_rails 0.1.0 → 0.3.0

data/lib/langsmithrb_rails/evaluation/evaluator.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+module LangsmithrbRails
+  module Evaluation
+    # Base evaluator class
+    class Evaluator
+      attr_reader :client, :project_name, :tags
+
+      # Initialize a new evaluator
+      # @param client [LangsmithrbRails::Client] LangSmith client
+      # @param project_name [String] Optional project name for evaluations
+      # @param tags [Array<String>] Optional tags for evaluations
+      def initialize(client: nil, project_name: nil, tags: [])
+        @client = client || LangsmithrbRails::Client.new
+        @project_name = project_name
+        @tags = tags
+      end
+
+      # Evaluate a prediction against a reference
+      # @param prediction [String, Hash] The prediction to evaluate
+      # @param reference [String, Hash] The reference to compare against
+      # @param input [Hash] Optional input that generated the prediction
+      # @return [Hash] Evaluation result with score and metadata
+      def evaluate(prediction, reference = nil, input = nil)
+        raise NotImplementedError, "Subclasses must implement evaluate method"
+      end
+
+      # Evaluate a run
+      # @param run_id [String] The ID of the run to evaluate
+      # @param reference [String, Hash] The reference to compare against
+      # @return [Hash] Evaluation result with score and metadata
+      def evaluate_run(run_id, reference = nil)
+        # Get the run
+        response = client.get_run(run_id)
+        
+        unless response[:status] >= 200 && response[:status] < 300
+          raise "Failed to get run: #{response[:error] || response[:body]}"
+        end
+        
+        run = response[:body]
+        
+        # Extract prediction from run outputs
+        prediction = extract_prediction_from_run(run)
+        
+        # Extract input from run inputs
+        input = run["inputs"]
+        
+        # Evaluate
+        result = evaluate(prediction, reference, input)
+        
+        # Create feedback
+        create_feedback(run_id, result)
+        
+        result
+      end
+
+      # Evaluate multiple runs
+      # @param run_ids [Array<String>] The IDs of the runs to evaluate
+      # @param references [Hash<String, Object>] Map of run IDs to references
+      # @return [Hash<String, Hash>] Map of run IDs to evaluation results
+      def evaluate_runs(run_ids, references = {})
+        results = {}
+        
+        run_ids.each do |run_id|
+          reference = references[run_id]
+          results[run_id] = evaluate_run(run_id, reference)
+        end
+        
+        results
+      end
+
+      # Evaluate a dataset
+      # @param dataset_id [String] The ID of the dataset to evaluate
+      # @param experiment_name [String] Name for the experiment
+      # @param target_llm [Object] Optional LLM to use for generating predictions
+      # @return [Hash] Evaluation results for the dataset
+      def evaluate_dataset(dataset_id, experiment_name, target_llm = nil)
+        # Get the dataset examples
+        response = client.list_examples(dataset_id)
+        
+        unless response[:status] >= 200 && response[:status] < 300
+          raise "Failed to get dataset examples: #{response[:error] || response[:body]}"
+        end
+        
+        examples = response[:body]
+        
+        results = {
+          experiment_name: experiment_name,
+          dataset_id: dataset_id,
+          evaluator_name: self.class.name,
+          results: []
+        }
+        
+        examples.each do |example|
+          # If target LLM is provided, generate a prediction
+          if target_llm
+            prediction = generate_prediction(target_llm, example["inputs"])
+          else
+            # Otherwise use the example's outputs as the prediction
+            prediction = example["outputs"]
+          end
+          
+          # Evaluate
+          result = evaluate(prediction, example["outputs"], example["inputs"])
+          
+          results[:results] << {
+            example_id: example["id"],
+            score: result[:score],
+            metadata: result[:metadata]
+          }
+        end
+        
+        results
+      end
+
+      private
+
+      # Extract prediction from run outputs
+      # @param run [Hash] The run data
+      # @return [Object] The prediction
+      def extract_prediction_from_run(run)
+        outputs = run["outputs"] || {}
+        
+        # Try common output keys
+        %w[output response result text completion answer].each do |key|
+          return outputs[key] if outputs.key?(key)
+        end
+        
+        # If no common keys found, return the entire outputs
+        outputs
+      end
+
+      # Generate a prediction using an LLM
+      # @param llm [Object] The LLM to use
+      # @param input [Hash] The input to generate from
+      # @return [Object] The generated prediction
+      def generate_prediction(llm, input)
+        if llm.respond_to?(:call)
+          llm.call(input)
+        elsif llm.respond_to?(:generate)
+          llm.generate(input)
+        elsif llm.respond_to?(:complete)
+          llm.complete(input)
+        else
+          raise "Unsupported LLM interface"
+        end
+      end
+
+      # Create feedback for a run
+      # @param run_id [String] The ID of the run
+      # @param result [Hash] The evaluation result
+      # @return [Hash] The created feedback
+      def create_feedback(run_id, result)
+        feedback_data = {
+          run_id: run_id,
+          key: self.class.name.split("::").last.downcase,
+          score: result[:score],
+          value: result[:metadata],
+          comment: result[:comment]
+        }.compact
+        
+        response = client.create_feedback(
+          feedback_data[:run_id],
+          feedback_data[:key],
+          feedback_data[:score],
+          value: feedback_data[:value],
+          comment: feedback_data[:comment]
+        )
+        
+        unless response[:status] >= 200 && response[:status] < 300
+          LangsmithrbRails.logger.error("Failed to create feedback: #{response[:error] || response[:body]}")
+        end
+        
+        response[:body]
+      end
+    end
+  end
+end

data/lib/langsmithrb_rails/evaluation/llm_evaluator.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require_relative "evaluator"
+
+module LangsmithrbRails
+  module Evaluation
+    # Evaluator that uses an LLM to evaluate responses
+    class LLMEvaluator < Evaluator
+      # Initialize a new LLM evaluator
+      # @param llm [Object] The LLM to use for evaluation
+      # @param criteria [String] Evaluation criteria
+      # @param client [LangsmithrbRails::Client] LangSmith client
+      # @param project_name [String] Optional project name for evaluations
+      # @param tags [Array<String>] Optional tags for evaluations
+      def initialize(llm:, criteria: nil, client: nil, project_name: nil, tags: [])
+        super(client: client, project_name: project_name, tags: tags)
+        @llm = llm
+        @criteria = criteria || "Evaluate the response for accuracy, relevance, and completeness."
+      end
+
+      # Evaluate a prediction against a reference
+      # @param prediction [String, Hash] The prediction to evaluate
+      # @param reference [String, Hash] The reference to compare against
+      # @param input [Hash] Optional input that generated the prediction
+      # @return [Hash] Evaluation result with score and metadata
+      def evaluate(prediction, reference = nil, input = nil)
+        # Extract strings
+        prediction_str = extract_string(prediction)
+        reference_str = extract_string(reference)
+        input_str = input.is_a?(Hash) ? input.to_json : input.to_s if input
+        
+        # Create evaluation prompt
+        prompt = create_evaluation_prompt(prediction_str, reference_str, input_str)
+        
+        # Get evaluation from LLM
+        begin
+          evaluation = get_llm_evaluation(prompt)
+          
+          # Parse the evaluation
+          score, feedback = parse_evaluation(evaluation)
+          
+          {
+            score: score,
+            metadata: {
+              feedback: feedback,
+              criteria: @criteria,
+              evaluation: evaluation
+            }
+          }
+        rescue => e
+          {
+            score: nil,
+            metadata: {
+              error: "Evaluation failed: #{e.message}"
+            }
+          }
+        end
+      end
+
+      private
+
+      # Extract a string from an object
+      # @param obj [Object] The object to extract a string from
+      # @return [String] The extracted string
+      def extract_string(obj)
+        return nil if obj.nil?
+        
+        case obj
+        when String
+          obj
+        when Hash
+          # Try common output keys
+          %w[output response result text completion answer].each do |key|
+            return obj[key].to_s if obj.key?(key)
+          end
+          obj.to_json
+        else
+          obj.to_s
+        end
+      end
+
+      # Create an evaluation prompt for the LLM
+      # @param prediction [String] The prediction string
+      # @param reference [String] The reference string
+      # @param input [String] The input string
+      # @return [String] The evaluation prompt
+      def create_evaluation_prompt(prediction, reference, input)
+        prompt = "You are an impartial evaluator. Your task is to evaluate the quality of a response.\n\n"
+        
+        if input
+          prompt += "Input:\n#{input}\n\n"
+        end
+        
+        prompt += "Response to evaluate:\n#{prediction}\n\n"
+        
+        if reference
+          prompt += "Reference (correct) response:\n#{reference}\n\n"
+        end
+        
+        prompt += "Evaluation criteria:\n#{@criteria}\n\n"
+        prompt += "Please provide a score between 0.0 and 1.0, where 1.0 is perfect, and detailed feedback.\n"
+        prompt += "Format your response as:\nScore: [score between 0.0 and 1.0]\nFeedback: [your detailed feedback]"
+        
+        prompt
+      end
+
+      # Get evaluation from LLM
+      # @param prompt [String] The evaluation prompt
+      # @return [String] The LLM's evaluation
+      # @raise [RuntimeError] If the LLM interface is not supported
+      def get_llm_evaluation(prompt)
+        result = if @llm.respond_to?(:call)
+          @llm.call(prompt)
+        elsif @llm.respond_to?(:generate)
+          @llm.generate(prompt)
+        elsif @llm.respond_to?(:complete)
+          @llm.complete(prompt)
+        elsif @llm.respond_to?(:chat)
+          @llm.chat(messages: [{ role: "user", content: prompt }])
+        else
+          raise "Unsupported LLM interface"
+        end
+        
+        # Extract content from response if it's a hash
+        if result.is_a?(Hash)
+          result[:content] || result["content"] || result[:text] || result["text"] || 
+          result[:completion] || result["completion"] || result.to_s
+        else
+          result.to_s
+        end
+      end
+
+      # Parse the LLM's evaluation
+      # @param evaluation [String] The LLM's evaluation
+      # @return [Array] Score and feedback
+      def parse_evaluation(evaluation)
+        # Extract score
+        score_match = evaluation.match(/Score:\s*(\d+(\.\d+)?)/)
+        score = score_match ? score_match[1].to_f : nil
+        
+        # Validate score
+        if score.nil? || score < 0.0 || score > 1.0
+          score = 0.5 # Default to middle score if invalid
+        end
+        
+        # Extract feedback
+        feedback_match = evaluation.match(/Feedback:\s*(.+)/m)
+        feedback = feedback_match ? feedback_match[1].strip : evaluation
+        
+        [score, feedback]
+      end
+    end
+  end
+end

data/lib/langsmithrb_rails/evaluation/string_evaluator.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require_relative "evaluator"
+
+module LangsmithrbRails
+  module Evaluation
+    # Evaluator for string comparisons
+    class StringEvaluator < Evaluator
+      # Initialize a new string evaluator
+      # @param match_type [Symbol] Type of string matching (:exact, :contains, :regex)
+      # @param case_sensitive [Boolean] Whether to perform case-sensitive matching
+      # @param client [LangsmithrbRails::Client] LangSmith client
+      # @param project_name [String] Optional project name for evaluations
+      # @param tags [Array<String>] Optional tags for evaluations
+      def initialize(match_type: :exact, case_sensitive: true, client: nil, project_name: nil, tags: [])
+        super(client: client, project_name: project_name, tags: tags)
+        @match_type = match_type
+        @case_sensitive = case_sensitive
+      end
+
+      # Evaluate a prediction against a reference
+      # @param prediction [String] The prediction to evaluate
+      # @param reference [String] The reference to compare against
+      # @param input [Hash] Optional input that generated the prediction
+      # @return [Hash] Evaluation result with score and metadata
+      def evaluate(prediction, reference = nil, input = nil)
+        # Extract strings from prediction and reference
+        prediction_str = extract_string(prediction)
+        reference_str = extract_string(reference)
+        
+        # If reference is nil, we can't evaluate
+        if reference_str.nil?
+          return {
+            score: nil,
+            metadata: {
+              error: "No reference provided for evaluation"
+            }
+          }
+        end
+        
+        # Prepare strings for comparison
+        unless @case_sensitive
+          prediction_str = prediction_str.downcase
+          reference_str = reference_str.downcase
+        end
+        
+        # Perform comparison based on match type
+        score, metadata = case @match_type
+                          when :exact
+                            exact_match(prediction_str, reference_str)
+                          when :contains
+                            contains_match(prediction_str, reference_str)
+                          when :regex
+                            regex_match(prediction_str, reference_str)
+                          else
+                            [0.0, { error: "Unknown match type: #{@match_type}" }]
+                          end
+        
+        {
+          score: score,
+          metadata: metadata
+        }
+      end
+
+      private
+
+      # Extract a string from an object
+      # @param obj [Object] The object to extract a string from
+      # @return [String] The extracted string
+      def extract_string(obj)
+        return nil if obj.nil?
+        
+        case obj
+        when String
+          obj
+        when Hash
+          # Try common output keys
+          %w[output response result text completion answer].each do |key|
+            return obj[key].to_s if obj.key?(key) || obj.key?(key.to_sym)
+          end
+          
+          # Try to find common keys between prediction and reference
+          if obj.key?(:prediction) && obj.key?(:reference)
+            return obj[:prediction].to_s
+          end
+          
+          # If it's a simple hash with a single value, use that
+          return obj.values.first.to_s if obj.size == 1
+          
+          # Otherwise convert the whole hash to a string
+          obj.to_s
+        else
+          obj.to_s
+        end
+      end
+
+      # Perform exact string matching
+      # @param prediction [String] The prediction string
+      # @param reference [String] The reference string
+      # @return [Array] Score and metadata
+      def exact_match(prediction, reference)
+        match = prediction == reference
+        [
+          match ? 1.0 : 0.0,
+          {
+            match: match,
+            match_type: "exact",
+            case_sensitive: @case_sensitive
+          }
+        ]
+      end
+
+      # Perform contains string matching
+      # @param prediction [String] The prediction string
+      # @param reference [String] The reference string
+      # @return [Array] Score and metadata
+      def contains_match(prediction, reference)
+        match = prediction.include?(reference)
+        [
+          match ? 1.0 : 0.0,
+          {
+            match: match,
+            match_type: "contains",
+            case_sensitive: @case_sensitive
+          }
+        ]
+      end
+
+      # Perform regex string matching
+      # @param prediction [String] The prediction string
+      # @param reference [String] The reference string (as regex pattern)
+      # @return [Array] Score and metadata
+      def regex_match(prediction, reference)
+        begin
+          regex = Regexp.new(reference, @case_sensitive ? nil : Regexp::IGNORECASE)
+          match = regex.match?(prediction)
+          [
+            match ? 1.0 : 0.0,
+            {
+              match: match,
+              match_type: "regex",
+              case_sensitive: @case_sensitive
+            }
+          ]
+        rescue RegexpError => e
+          [
+            0.0,
+            {
+              error: "Invalid regex pattern: #{e.message}",
+              match_type: "regex",
+              case_sensitive: @case_sensitive
+            }
+          ]
+        end
+      end
+    end
+  end
+end

data/lib/langsmithrb_rails/evaluation.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+require_relative "evaluation/evaluator"
+require_relative "evaluation/string_evaluator"
+require_relative "evaluation/llm_evaluator"
+
+module LangsmithrbRails
+  # Evaluation framework for LangSmith
+  module Evaluation
+    # Create a new evaluator
+    # @param type [Symbol] Type of evaluator (:string, :llm)
+    # @param options [Hash] Options for the evaluator
+    # @return [Evaluator] The evaluator instance
+    def self.create(type, **options)
+      case type
+      when :string
+        StringEvaluator.new(**options)
+      when :llm
+        unless options[:llm]
+          raise ArgumentError, "LLM evaluator requires an :llm option"
+        end
+        LLMEvaluator.new(**options)
+      else
+        raise ArgumentError, "Unknown evaluator type: #{type}"
+      end
+    end
+
+    # Run a batch evaluation on a dataset
+    # @param dataset_id [String] The dataset ID to evaluate
+    # @param evaluators [Array<Evaluator>] The evaluators to use
+    # @param experiment_name [String] Name for the experiment
+    # @param target_llm [Object] Optional LLM to use for generating predictions
+    # @return [Hash] Evaluation results
+    def self.evaluate_dataset(dataset_id, evaluators, experiment_name: nil, target_llm: nil)
+      experiment_name ||= "Evaluation #{Time.now.utc.iso8601}"
+      
+      results = {
+        experiment_name: experiment_name,
+        dataset_id: dataset_id,
+        evaluators: evaluators.map { |e| e.class.name },
+        results: []
+      }
+      
+      evaluators.each do |evaluator|
+        evaluator_results = evaluator.evaluate_dataset(dataset_id, experiment_name, target_llm)
+        results[:results] << evaluator_results
+      end
+      
+      results
+    end
+
+    # Run a batch evaluation on runs
+    # @param run_ids [Array<String>] The run IDs to evaluate
+    # @param evaluators [Array<Evaluator>] The evaluators to use
+    # @param references [Hash<String, Object>] Map of run IDs to references
+    # @return [Hash] Evaluation results
+    def self.evaluate_runs(run_ids, evaluators, references = {})
+      results = {
+        run_ids: run_ids,
+        evaluators: evaluators.map { |e| e.class.name },
+        results: {}
+      }
+      
+      run_ids.each do |run_id|
+        results[:results][run_id] = {}
+        
+        evaluators.each do |evaluator|
+          reference = references[run_id]
+          results[:results][run_id][evaluator.class.name] = evaluator.evaluate_run(run_id, reference)
+        end
+      end
+      
+      results
+    end
+  end
+end

data/lib/langsmithrb_rails/generators/langsmithrb_rails/langsmith_generator.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module LangsmithrbRails
+  module Generators
+    # Generator for adding LangSmith support to a Rails application
+    class LangsmithGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+      desc "Adds LangSmith support to your Rails application"
+
+      def create_initializer
+        template "langsmith_initializer.rb", "config/initializers/langsmith.rb"
+      end
+
+      def update_env_example
+        create_file ".env.example" unless File.exist?(".env.example")
+        
+        # Check if LangSmith config is already in the file
+        env_content = File.exist?(".env.example") ? File.read(".env.example") : ""
+        return if env_content.include?("LANGSMITH_API_KEY")
+        
+        append_to_file ".env.example" do
+          # Add a newline if the file doesn't end with one
+          (env_content.end_with?("\n") ? "" : "\n") + 
+          <<~ENV
+            # LangSmith configuration
+            LANGSMITH_API_KEY=
+            LANGSMITH_PROJECT=
+          ENV
+        end
+      end
+
+      def update_gitignore
+        create_file ".gitignore" unless File.exist?(".gitignore")
+        
+        # Check if .env is already in gitignore
+        gitignore_content = File.exist?(".gitignore") ? File.read(".gitignore") : ""
+        return if gitignore_content.match?(/^\.env\s*$/)
+        
+        append_to_file ".gitignore" do
+          # Add a newline if the file doesn't end with one
+          (gitignore_content.end_with?("\n") ? "" : "\n") + 
+          <<~GITIGNORE
+            # LangSmith
+            .env
+          GITIGNORE
+        end
+      end
+
+      def display_post_install_message
+        say "\n🎉 LangSmith support has been added to your Rails application!", :green
+        say "\nTo enable LangSmith tracing, add the following to your .env file:", :yellow
+        say "LANGSMITH_API_KEY=your_api_key", :yellow
+        say "LANGSMITH_PROJECT=your_project_name (optional)", :yellow
+        say "\nYou can get your API key from https://smith.langchain.com/", :yellow
+        say "\nRestart your Rails server to apply the changes.", :yellow
+      end
+    end
+  end
+end

data/lib/langsmithrb_rails/generators/langsmithrb_rails/templates/langsmith_initializer.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# Configure LangSmith using the langsmithrb gem
+
+# First, configure the langsmithrb gem directly
+Langsmithrb.configure do |config|
+  config.api_key = ENV["LANGSMITH_API_KEY"]
+  config.project_name = ENV["LANGSMITH_PROJECT"]
+  config.tracing_enabled = ENV["LANGSMITH_API_KEY"].present?
+end
+
+# Then, configure our Rails integration
+LangsmithrbRails.configure do |config|
+  # Enable LangSmith tracing
+  config.enabled = ENV["LANGSMITH_API_KEY"].present?
+  
+  # Your LangSmith API key from https://smith.langchain.com/
+  config.api_key = ENV["LANGSMITH_API_KEY"]
+  
+  # Optional: The project name to use for LangSmith traces
+  config.project_name = ENV["LANGSMITH_PROJECT"]
+end

data/lib/langsmithrb_rails/langsmith.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "langsmithrb"
+
+module LangsmithrbRails
+  # LangSmith integration for Rails applications using the langsmithrb gem
+  module LangSmith
+    class << self
+      # Configure LangSmith tracing
+      # This method sets up LangSmith tracing using the langsmithrb gem
+      # @param api_key [String] LangSmith API key
+      # @param project_name [String] Optional project name for LangSmith traces
+      # @param tracing [Boolean] Whether to enable tracing (default: true)
+      def configure(api_key:, project_name: nil, tracing: true)
+        Langsmithrb.configure do |config|
+          config.api_key = api_key
+          config.project_name = project_name if project_name
+          config.tracing_enabled = tracing
+        end
+      end
+
+      # Check if LangSmith tracing is enabled
+      # @return [Boolean] Whether LangSmith tracing is enabled
+      def enabled?
+        Langsmithrb.config.tracing_enabled && !Langsmithrb.config.api_key.nil?
+      end
+
+      # Get the current LangSmith project name
+      # @return [String, nil] The current LangSmith project name or nil if not set
+      def project_name
+        Langsmithrb.config.project_name
+      end
+    end
+  end
+end