langsmithrb_rails 0.1.0 → 0.1.1

data/lib/generators/langsmithrb_rails/demo/templates/llm_service.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+# Service for interacting with LLM providers
+class LlmService
+  include LangsmithrbRails::TracedService
+  
+  # Initialize the service
+  # @param provider [String] LLM provider to use (openai, anthropic, or mock)
+  def initialize(provider = ENV.fetch("LLM_PROVIDER", "<%= options[:provider] %>"))
+    @provider = provider.to_s.downcase
+  end
+  
+  # Generate a response to a prompt
+  # @param prompt [String] User prompt
+  # @param context [Array<Hash>] Previous messages for context
+  # @param options [Hash] Additional options
+  # @return [String] Generated response
+  def generate(prompt, context = [], options = {})
+    # Create trace with LangSmith
+    langsmith_trace("llm_generate", inputs: { prompt: prompt, context: context }) do |run|
+      response = case @provider
+                 when "openai"
+                   generate_with_openai(prompt, context, options)
+                 when "anthropic"
+                   generate_with_anthropic(prompt, context, options)
+                 else
+                   generate_with_mock(prompt, context, options)
+                 end
+      
+      # Record the output
+      run.outputs = { response: response }
+      
+      response
+    end
+  end
+  
+  private
+  
+  # Generate with OpenAI
+  # @param prompt [String] User prompt
+  # @param context [Array<Hash>] Previous messages for context
+  # @param options [Hash] Additional options
+  # @return [String] Generated response
+  def generate_with_openai(prompt, context = [], options = {})
+    require "openai"
+    
+    client = OpenAI::Client.new(access_token: ENV.fetch("OPENAI_API_KEY"))
+    
+    # Format messages for OpenAI
+    messages = format_openai_messages(prompt, context)
+    
+    # Call OpenAI API with LangSmith tracing
+    langsmith_trace("openai_chat_completion", 
+                    inputs: { messages: messages },
+                    run_type: "llm") do |run|
+      response = client.chat(
+        parameters: {
+          model: options[:model] || ENV.fetch("OPENAI_MODEL", "gpt-3.5-turbo"),
+          messages: messages,
+          temperature: options[:temperature] || 0.7,
+          max_tokens: options[:max_tokens] || 1000
+        }
+      )
+      
+      # Record the API response
+      run.outputs = { response: response }
+      
+      # Extract and return the text
+      response.dig("choices", 0, "message", "content")
+    end
+  rescue => e
+    Rails.logger.error("OpenAI API error: #{e.message}")
+    "Sorry, I encountered an error: #{e.message}"
+  end
+  
+  # Generate with Anthropic
+  # @param prompt [String] User prompt
+  # @param context [Array<Hash>] Previous messages for context
+  # @param options [Hash] Additional options
+  # @return [String] Generated response
+  def generate_with_anthropic(prompt, context = [], options = {})
+    require "anthropic"
+    
+    client = Anthropic::Client.new(api_key: ENV.fetch("ANTHROPIC_API_KEY"))
+    
+    # Format messages for Anthropic
+    messages = format_anthropic_messages(prompt, context)
+    
+    # Call Anthropic API with LangSmith tracing
+    langsmith_trace("anthropic_completion", 
+                    inputs: { messages: messages },
+                    run_type: "llm") do |run|
+      response = client.messages.create(
+        model: options[:model] || ENV.fetch("ANTHROPIC_MODEL", "claude-2"),
+        max_tokens: options[:max_tokens] || 1000,
+        temperature: options[:temperature] || 0.7,
+        messages: messages
+      )
+      
+      # Record the API response
+      run.outputs = { response: response }
+      
+      # Extract and return the text
+      response.content.first.text
+    end
+  rescue => e
+    Rails.logger.error("Anthropic API error: #{e.message}")
+    "Sorry, I encountered an error: #{e.message}"
+  end
+  
+  # Generate with mock (for testing)
+  # @param prompt [String] User prompt
+  # @param context [Array<Hash>] Previous messages for context
+  # @param options [Hash] Additional options
+  # @return [String] Generated response
+  def generate_with_mock(prompt, context = [], options = {})
+    # Simulate a delay
+    sleep(0.5)
+    
+    # Generate a mock response
+    "This is a mock response to: \"#{prompt}\"\n\nIn a real application, this would be generated by an LLM API."
+  end
+  
+  # Format messages for OpenAI
+  # @param prompt [String] User prompt
+  # @param context [Array<Hash>] Previous messages for context
+  # @return [Array<Hash>] Formatted messages
+  def format_openai_messages(prompt, context)
+    messages = []
+    
+    # Add system message if provided
+    system_message = ENV["OPENAI_SYSTEM_MESSAGE"]
+    messages << { role: "system", content: system_message } if system_message.present?
+    
+    # Add context messages
+    context.each do |message|
+      role = message[:role] || (message[:is_user] ? "user" : "assistant")
+      messages << { role: role, content: message[:content] }
+    end
+    
+    # Add the current prompt
+    messages << { role: "user", content: prompt }
+    
+    messages
+  end
+  
+  # Format messages for Anthropic
+  # @param prompt [String] User prompt
+  # @param context [Array<Hash>] Previous messages for context
+  # @return [Array<Hash>] Formatted messages
+  def format_anthropic_messages(prompt, context)
+    messages = []
+    
+    # Add context messages
+    context.each do |message|
+      role = message[:is_user] ? "user" : "assistant"
+      messages << { role: role, content: message[:content] }
+    end
+    
+    # Add the current prompt
+    messages << { role: "user", content: prompt }
+    
+    messages
+  end
+end

data/lib/generators/langsmithrb_rails/evals/evals_generator.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module LangsmithrbRails
+  module Generators
+    # Generator for adding LangSmith evaluations to Rails applications
+    class EvalsGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+      
+      desc "Adds LangSmith evaluations to your Rails application"
+      
+      def create_directories
+        empty_directory "config/langsmith/evals/datasets"
+        empty_directory "config/langsmith/evals/checks"
+        empty_directory "config/langsmith/evals/targets"
+      end
+      
+      def create_sample_dataset
+        template "datasets/sample.yml", "config/langsmith/evals/datasets/sample.yml"
+      end
+      
+      def create_sample_check
+        template "checks/correctness.rb", "config/langsmith/evals/checks/correctness.rb"
+        template "checks/llm_graded.rb", "config/langsmith/evals/checks/llm_graded.rb"
+      end
+      
+      def create_sample_target
+        template "targets/http.rb", "config/langsmith/evals/targets/http.rb"
+        template "targets/ruby.rb", "config/langsmith/evals/targets/ruby.rb"
+      end
+      
+      def create_rake_task
+        template "langsmith_evals.rake", "lib/tasks/langsmith_evals.rake"
+      end
+      
+      def display_post_install_message
+        say "\n"
+        say "LangSmith evaluations have been added to your Rails application! 🎉", :green
+        say "\n"
+        say "Usage:", :yellow
+        say "  1. Customize the sample dataset in config/langsmith/evals/datasets/", :yellow
+        say "  2. Run an evaluation:", :yellow
+        say "     bin/rails langsmith:eval[sample,http,my_experiment]", :yellow
+        say "  3. Compare experiments:", :yellow
+        say "     bin/rails langsmith:compare[exp_a,exp_b]", :yellow
+        say "\n"
+        say "To add CI integration for evaluations, run:", :yellow
+        say "  bin/rails g langsmithrb_rails:ci", :yellow
+        say "\n"
+      end
+    end
+  end
+end

data/lib/generators/langsmithrb_rails/evals/templates/checks/correctness.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module LangsmithrbRails
+  module Evals
+    module Checks
+      # Simple correctness check for evaluating LLM responses
+      class Correctness
+        # Check if the response is correct
+        # @param input [Hash] Input data
+        # @param response [Hash] Response data
+        # @param expected [Hash] Expected output data
+        # @return [Hash] Evaluation result
+        def self.evaluate(input, response, expected)
+          result = {
+            score: 0.0,
+            reasoning: "",
+            passed: false
+          }
+          
+          # Extract the answer from the response
+          answer = extract_answer(response)
+          
+          # Check for exact match
+          if expected["answer"] && answer == expected["answer"]
+            result[:score] = 1.0
+            result[:reasoning] = "Exact match with expected answer"
+            result[:passed] = true
+            return result
+          end
+          
+          # Check for partial matches using contains
+          if expected["answer_contains"] && expected["answer_contains"].is_a?(Array)
+            matches = expected["answer_contains"].select { |phrase| answer.include?(phrase) }
+            match_ratio = matches.size.to_f / expected["answer_contains"].size
+            
+            result[:score] = match_ratio
+            result[:reasoning] = "Matched #{matches.size}/#{expected["answer_contains"].size} expected phrases"
+            result[:passed] = match_ratio >= 0.5
+            return result
+          end
+          
+          # Check for code snippets
+          if expected["code_contains"] && expected["code_contains"].is_a?(Array)
+            matches = expected["code_contains"].select { |phrase| answer.include?(phrase) }
+            match_ratio = matches.size.to_f / expected["code_contains"].size
+            
+            result[:score] = match_ratio
+            result[:reasoning] = "Code snippet matched #{matches.size}/#{expected["code_contains"].size} expected elements"
+            result[:passed] = match_ratio >= 0.5
+            return result
+          end
+          
+          # No match found
+          result[:reasoning] = "No matching criteria found"
+          result
+        end
+        
+        # Extract the answer from the response
+        # @param response [Hash] Response data
+        # @return [String] Extracted answer
+        def self.extract_answer(response)
+          return response["answer"] if response["answer"]
+          return response["text"] if response["text"]
+          return response["content"] if response["content"]
+          return response["output"] if response["output"]
+          return response.to_s
+        end
+      end
+    end
+  end
+end

data/lib/generators/langsmithrb_rails/evals/templates/checks/llm_graded.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module LangsmithrbRails
+  module Evals
+    module Checks
+      # LLM-based grading for evaluating responses
+      class LlmGraded
+        # Check if the response is correct using an LLM
+        # @param input [Hash] Input data
+        # @param response [Hash] Response data
+        # @param expected [Hash] Expected output data
+        # @return [Hash] Evaluation result
+        def self.evaluate(input, response, expected)
+          result = {
+            score: 0.0,
+            reasoning: "",
+            passed: false
+          }
+          
+          # Extract the answer from the response
+          answer = extract_answer(response)
+          expected_answer = extract_answer(expected)
+          
+          # Create the prompt for the LLM
+          prompt = create_grading_prompt(input, answer, expected_answer)
+          
+          # Call the LLM for grading
+          llm_response = call_llm(prompt)
+          
+          # Parse the LLM response
+          parsed_result = parse_llm_response(llm_response)
+          
+          # Update the result with the parsed data
+          result[:score] = parsed_result[:score]
+          result[:reasoning] = parsed_result[:reasoning]
+          result[:passed] = parsed_result[:score] >= 0.7
+          
+          result
+        end
+        
+        # Extract the answer from the response
+        # @param response [Hash] Response data
+        # @return [String] Extracted answer
+        def self.extract_answer(response)
+          return response["answer"] if response["answer"]
+          return response["text"] if response["text"]
+          return response["content"] if response["content"]
+          return response["output"] if response["output"]
+          return response.to_s
+        end
+        
+        # Create a prompt for the LLM to grade the response
+        # @param input [Hash] Input data
+        # @param answer [String] Actual answer
+        # @param expected_answer [String] Expected answer
+        # @return [String] Prompt for the LLM
+        def self.create_grading_prompt(input, answer, expected_answer)
+          <<~PROMPT
+            You are an expert evaluator. Your task is to grade the quality and correctness of a response.
+            
+            Question: #{input["question"]}
+            
+            Expected Answer: #{expected_answer}
+            
+            Actual Response: #{answer}
+            
+            Please evaluate the response based on:
+            1. Correctness: Is the information accurate?
+            2. Completeness: Does it fully address the question?
+            3. Clarity: Is it well-explained and easy to understand?
+            
+            Provide your evaluation in the following format:
+            
+            Score: [a number between 0.0 and 1.0]
+            Reasoning: [your detailed explanation]
+          PROMPT
+        end
+        
+        # Call the LLM for grading
+        # @param prompt [String] Prompt for the LLM
+        # @return [String] LLM response
+        def self.call_llm(prompt)
+          # Check if OpenAI is configured
+          if defined?(OpenAI) && ENV["OPENAI_API_KEY"].present?
+            client = OpenAI::Client.new(access_token: ENV["OPENAI_API_KEY"])
+            response = client.chat(
+              parameters: {
+                model: ENV.fetch("LANGSMITH_EVAL_MODEL", "gpt-3.5-turbo"),
+                messages: [{ role: "user", content: prompt }],
+                temperature: 0.0
+              }
+            )
+            return response.dig("choices", 0, "message", "content")
+          end
+          
+          # Check if Anthropic is configured
+          if defined?(Anthropic) && ENV["ANTHROPIC_API_KEY"].present?
+            client = Anthropic::Client.new(api_key: ENV["ANTHROPIC_API_KEY"])
+            response = client.messages.create(
+              model: ENV.fetch("LANGSMITH_EVAL_MODEL", "claude-2"),
+              max_tokens: 1024,
+              messages: [{ role: "user", content: prompt }]
+            )
+            return response.content.first.text
+          end
+          
+          # Fall back to a simple evaluation
+          "Score: 0.5\nReasoning: Unable to perform LLM-based evaluation. Please configure an LLM provider."
+        end
+        
+        # Parse the LLM response
+        # @param response [String] LLM response
+        # @return [Hash] Parsed result
+        def self.parse_llm_response(response)
+          result = {
+            score: 0.5,
+            reasoning: "Unable to parse LLM response"
+          }
+          
+          # Extract score
+          if response =~ /Score:\s*([\d\.]+)/i
+            result[:score] = $1.to_f
+            # Ensure score is between 0 and 1
+            result[:score] = [0.0, [1.0, result[:score]].min].max
+          end
+          
+          # Extract reasoning
+          if response =~ /Reasoning:\s*(.+)/im
+            result[:reasoning] = $1.strip
+          end
+          
+          result
+        end
+      end
+    end
+  end
+end

data/lib/generators/langsmithrb_rails/evals/templates/datasets/sample.yml

@@ -0,0 +1,60 @@
+# Sample evaluation dataset
+# This file defines a set of inputs and expected outputs for LLM evaluation
+
+name: sample_dataset
+description: A sample dataset for evaluating LLM responses
+version: 1.0.0
+
+# Each item in the dataset represents a test case
+items:
+  - id: question_1
+    input:
+      question: "What is the capital of France?"
+    expected_output:
+      answer: "Paris"
+    metadata:
+      category: geography
+      difficulty: easy
+
+  - id: question_2
+    input:
+      question: "Who wrote 'Pride and Prejudice'?"
+    expected_output:
+      answer: "Jane Austen"
+    metadata:
+      category: literature
+      difficulty: easy
+
+  - id: question_3
+    input:
+      question: "What is the formula for calculating the area of a circle?"
+    expected_output:
+      answer: "A = πr²"
+    metadata:
+      category: mathematics
+      difficulty: medium
+
+  - id: question_4
+    input:
+      question: "Explain how a transformer neural network works."
+    expected_output:
+      answer_contains:
+        - "attention mechanism"
+        - "self-attention"
+        - "encoder"
+        - "decoder"
+    metadata:
+      category: machine_learning
+      difficulty: hard
+
+  - id: question_5
+    input:
+      question: "Write a function to check if a string is a palindrome."
+    expected_output:
+      code_contains:
+        - "function"
+        - "return"
+        - "reverse"
+    metadata:
+      category: programming
+      difficulty: medium