langsmithrb_rails 0.1.0 → 0.3.0

data/lib/generators/langsmithrb_rails/demo/templates/index.html.erb

@@ -0,0 +1,180 @@
+<%# Chat demo index page %>
+<div class="chat-container" data-controller="chat">
+  <div class="chat-header">
+    <h1>LangSmith Rails Demo</h1>
+    <p class="subtitle">A simple chat interface with LangSmith tracing</p>
+  </div>
+  
+  <div class="chat-messages" id="chat-messages">
+    <% if @messages.empty? %>
+      <div class="welcome-message">
+        <h2>Welcome to the LangSmith Rails Demo!</h2>
+        <p>This is a simple chat interface that demonstrates LangSmith tracing in a Rails application.</p>
+        <p>Every message you send will be traced with LangSmith, allowing you to monitor and debug your LLM interactions.</p>
+        <p>Try asking a question below to get started!</p>
+      </div>
+    <% else %>
+      <% @messages.each do |message| %>
+        <%= render partial: "message", locals: { message: message } %>
+      <% end %>
+    <% end %>
+  </div>
+  
+  <div class="chat-form">
+    <%= form_with(model: @message, url: chat_index_path, data: { action: "submit->chat#submit" }) do |form| %>
+      <div class="input-group">
+        <%= form.text_area :content, placeholder: "Type your message...", class: "chat-input", data: { chat_target: "input" } %>
+        <button type="submit" class="send-button" data-chat-target="submitButton">
+          Send
+        </button>
+      </div>
+    <% end %>
+  </div>
+  
+  <div class="chat-footer">
+    <p>
+      Powered by LangSmith Rails
+      <% if ENV["LANGSMITH_PROJECT"].present? %>
+        | Project: <%= ENV["LANGSMITH_PROJECT"] %>
+      <% end %>
+    </p>
+  </div>
+</div>
+
+<%# Partial for rendering a message %>
+<% content_for :partials do %>
+  <script type="text/html" id="message-template">
+    <div class="message <%= message.is_user? ? 'user-message' : 'assistant-message' %>">
+      <div class="message-content">
+        <%= message.content %>
+      </div>
+      <div class="message-meta">
+        <span class="message-time"><%= message.created_at.strftime("%H:%M") %></span>
+        <span class="message-role"><%= message.is_user? ? "You" : "Assistant" %></span>
+      </div>
+    </div>
+  </script>
+<% end %>
+
+<style>
+  .chat-container {
+    max-width: 800px;
+    margin: 0 auto;
+    padding: 20px;
+    font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu, Cantarell, "Open Sans", "Helvetica Neue", sans-serif;
+  }
+  
+  .chat-header {
+    text-align: center;
+    margin-bottom: 20px;
+    padding-bottom: 10px;
+    border-bottom: 1px solid #eaeaea;
+  }
+  
+  .chat-header h1 {
+    margin-bottom: 5px;
+    color: #333;
+  }
+  
+  .subtitle {
+    color: #666;
+    margin-top: 0;
+  }
+  
+  .chat-messages {
+    height: 500px;
+    overflow-y: auto;
+    padding: 10px;
+    background-color: #f9f9f9;
+    border-radius: 8px;
+    margin-bottom: 20px;
+  }
+  
+  .welcome-message {
+    text-align: center;
+    padding: 40px 20px;
+    color: #666;
+  }
+  
+  .welcome-message h2 {
+    color: #333;
+    margin-bottom: 15px;
+  }
+  
+  .message {
+    margin-bottom: 15px;
+    padding: 10px 15px;
+    border-radius: 8px;
+    max-width: 80%;
+    position: relative;
+  }
+  
+  .user-message {
+    background-color: #e1f5fe;
+    margin-left: auto;
+    border-bottom-right-radius: 0;
+  }
+  
+  .assistant-message {
+    background-color: #f0f0f0;
+    margin-right: auto;
+    border-bottom-left-radius: 0;
+  }
+  
+  .message-content {
+    white-space: pre-wrap;
+  }
+  
+  .message-meta {
+    font-size: 0.8em;
+    color: #999;
+    margin-top: 5px;
+    text-align: right;
+  }
+  
+  .chat-form {
+    margin-bottom: 20px;
+  }
+  
+  .input-group {
+    display: flex;
+    gap: 10px;
+  }
+  
+  .chat-input {
+    flex-grow: 1;
+    padding: 10px;
+    border: 1px solid #ddd;
+    border-radius: 4px;
+    resize: none;
+    height: 60px;
+    font-family: inherit;
+  }
+  
+  .send-button {
+    padding: 10px 20px;
+    background-color: #2196f3;
+    color: white;
+    border: none;
+    border-radius: 4px;
+    cursor: pointer;
+    transition: background-color 0.2s;
+  }
+  
+  .send-button:hover {
+    background-color: #0b7dda;
+  }
+  
+  .send-button:disabled {
+    background-color: #cccccc;
+    cursor: not-allowed;
+  }
+  
+  .chat-footer {
+    text-align: center;
+    color: #999;
+    font-size: 0.8em;
+    padding-top: 10px;
+    border-top: 1px solid #eaeaea;
+  }
+</style>

data/lib/generators/langsmithrb_rails/demo/templates/llm_service.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+# Service for interacting with LLM providers
+class LlmService
+  include LangsmithrbRails::TracedService
+  
+  # Initialize the service
+  # @param provider [String] LLM provider to use (openai, anthropic, or mock)
+  def initialize(provider = ENV.fetch("LLM_PROVIDER", "<%= options[:provider] %>"))
+    @provider = provider.to_s.downcase
+  end
+  
+  # Generate a response to a prompt
+  # @param prompt [String] User prompt
+  # @param context [Array<Hash>] Previous messages for context
+  # @param options [Hash] Additional options
+  # @return [String] Generated response
+  def generate(prompt, context = [], options = {})
+    # Create trace with LangSmith
+    langsmith_trace("llm_generate", inputs: { prompt: prompt, context: context }) do |run|
+      response = case @provider
+                 when "openai"
+                   generate_with_openai(prompt, context, options)
+                 when "anthropic"
+                   generate_with_anthropic(prompt, context, options)
+                 else
+                   generate_with_mock(prompt, context, options)
+                 end
+      
+      # Record the output
+      run.outputs = { response: response }
+      
+      response
+    end
+  end
+  
+  private
+  
+  # Generate with OpenAI
+  # @param prompt [String] User prompt
+  # @param context [Array<Hash>] Previous messages for context
+  # @param options [Hash] Additional options
+  # @return [String] Generated response
+  def generate_with_openai(prompt, context = [], options = {})
+    require "openai"
+    
+    client = OpenAI::Client.new(access_token: ENV.fetch("OPENAI_API_KEY"))
+    
+    # Format messages for OpenAI
+    messages = format_openai_messages(prompt, context)
+    
+    # Call OpenAI API with LangSmith tracing
+    langsmith_trace("openai_chat_completion", 
+                    inputs: { messages: messages },
+                    run_type: "llm") do |run|
+      response = client.chat(
+        parameters: {
+          model: options[:model] || ENV.fetch("OPENAI_MODEL", "gpt-3.5-turbo"),
+          messages: messages,
+          temperature: options[:temperature] || 0.7,
+          max_tokens: options[:max_tokens] || 1000
+        }
+      )
+      
+      # Record the API response
+      run.outputs = { response: response }
+      
+      # Extract and return the text
+      response.dig("choices", 0, "message", "content")
+    end
+  rescue => e
+    Rails.logger.error("OpenAI API error: #{e.message}")
+    "Sorry, I encountered an error: #{e.message}"
+  end
+  
+  # Generate with Anthropic
+  # @param prompt [String] User prompt
+  # @param context [Array<Hash>] Previous messages for context
+  # @param options [Hash] Additional options
+  # @return [String] Generated response
+  def generate_with_anthropic(prompt, context = [], options = {})
+    require "anthropic"
+    
+    client = Anthropic::Client.new(api_key: ENV.fetch("ANTHROPIC_API_KEY"))
+    
+    # Format messages for Anthropic
+    messages = format_anthropic_messages(prompt, context)
+    
+    # Call Anthropic API with LangSmith tracing
+    langsmith_trace("anthropic_completion", 
+                    inputs: { messages: messages },
+                    run_type: "llm") do |run|
+      response = client.messages.create(
+        model: options[:model] || ENV.fetch("ANTHROPIC_MODEL", "claude-2"),
+        max_tokens: options[:max_tokens] || 1000,
+        temperature: options[:temperature] || 0.7,
+        messages: messages
+      )
+      
+      # Record the API response
+      run.outputs = { response: response }
+      
+      # Extract and return the text
+      response.content.first.text
+    end
+  rescue => e
+    Rails.logger.error("Anthropic API error: #{e.message}")
+    "Sorry, I encountered an error: #{e.message}"
+  end
+  
+  # Generate with mock (for testing)
+  # @param prompt [String] User prompt
+  # @param context [Array<Hash>] Previous messages for context
+  # @param options [Hash] Additional options
+  # @return [String] Generated response
+  def generate_with_mock(prompt, context = [], options = {})
+    # Simulate a delay
+    sleep(0.5)
+    
+    # Generate a mock response
+    "This is a mock response to: \"#{prompt}\"\n\nIn a real application, this would be generated by an LLM API."
+  end
+  
+  # Format messages for OpenAI
+  # @param prompt [String] User prompt
+  # @param context [Array<Hash>] Previous messages for context
+  # @return [Array<Hash>] Formatted messages
+  def format_openai_messages(prompt, context)
+    messages = []
+    
+    # Add system message if provided
+    system_message = ENV["OPENAI_SYSTEM_MESSAGE"]
+    messages << { role: "system", content: system_message } if system_message.present?
+    
+    # Add context messages
+    context.each do |message|
+      role = message[:role] || (message[:is_user] ? "user" : "assistant")
+      messages << { role: role, content: message[:content] }
+    end
+    
+    # Add the current prompt
+    messages << { role: "user", content: prompt }
+    
+    messages
+  end
+  
+  # Format messages for Anthropic
+  # @param prompt [String] User prompt
+  # @param context [Array<Hash>] Previous messages for context
+  # @return [Array<Hash>] Formatted messages
+  def format_anthropic_messages(prompt, context)
+    messages = []
+    
+    # Add context messages
+    context.each do |message|
+      role = message[:is_user] ? "user" : "assistant"
+      messages << { role: role, content: message[:content] }
+    end
+    
+    # Add the current prompt
+    messages << { role: "user", content: prompt }
+    
+    messages
+  end
+end

data/lib/generators/langsmithrb_rails/evals/evals_generator.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module LangsmithrbRails
+  module Generators
+    # Generator for adding LangSmith evaluations to Rails applications
+    class EvalsGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+      
+      desc "Adds LangSmith evaluations to your Rails application"
+      
+      def create_directories
+        empty_directory "config/langsmith/evals/datasets"
+        empty_directory "config/langsmith/evals/checks"
+        empty_directory "config/langsmith/evals/targets"
+      end
+      
+      def create_sample_dataset
+        template "datasets/sample.yml", "config/langsmith/evals/datasets/sample.yml"
+      end
+      
+      def create_sample_check
+        template "checks/correctness.rb", "config/langsmith/evals/checks/correctness.rb"
+        template "checks/llm_graded.rb", "config/langsmith/evals/checks/llm_graded.rb"
+      end
+      
+      def create_sample_target
+        template "targets/http.rb", "config/langsmith/evals/targets/http.rb"
+        template "targets/ruby.rb", "config/langsmith/evals/targets/ruby.rb"
+      end
+      
+      def create_rake_task
+        template "langsmith_evals.rake", "lib/tasks/langsmith_evals.rake"
+      end
+      
+      def display_post_install_message
+        say "\n"
+        say "LangSmith evaluations have been added to your Rails application! 🎉", :green
+        say "\n"
+        say "Usage:", :yellow
+        say "  1. Customize the sample dataset in config/langsmith/evals/datasets/", :yellow
+        say "  2. Run an evaluation:", :yellow
+        say "     bin/rails langsmith:eval[sample,http,my_experiment]", :yellow
+        say "  3. Compare experiments:", :yellow
+        say "     bin/rails langsmith:compare[exp_a,exp_b]", :yellow
+        say "\n"
+        say "To add CI integration for evaluations, run:", :yellow
+        say "  bin/rails g langsmithrb_rails:ci", :yellow
+        say "\n"
+      end
+    end
+  end
+end

data/lib/generators/langsmithrb_rails/evals/templates/checks/correctness.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module LangsmithrbRails
+  module Evals
+    module Checks
+      # Simple correctness check for evaluating LLM responses
+      class Correctness
+        # Check if the response is correct
+        # @param input [Hash] Input data
+        # @param response [Hash] Response data
+        # @param expected [Hash] Expected output data
+        # @return [Hash] Evaluation result
+        def self.evaluate(input, response, expected)
+          result = {
+            score: 0.0,
+            reasoning: "",
+            passed: false
+          }
+          
+          # Extract the answer from the response
+          answer = extract_answer(response)
+          
+          # Check for exact match
+          if expected["answer"] && answer == expected["answer"]
+            result[:score] = 1.0
+            result[:reasoning] = "Exact match with expected answer"
+            result[:passed] = true
+            return result
+          end
+          
+          # Check for partial matches using contains
+          if expected["answer_contains"] && expected["answer_contains"].is_a?(Array)
+            matches = expected["answer_contains"].select { |phrase| answer.include?(phrase) }
+            match_ratio = matches.size.to_f / expected["answer_contains"].size
+            
+            result[:score] = match_ratio
+            result[:reasoning] = "Matched #{matches.size}/#{expected["answer_contains"].size} expected phrases"
+            result[:passed] = match_ratio >= 0.5
+            return result
+          end
+          
+          # Check for code snippets
+          if expected["code_contains"] && expected["code_contains"].is_a?(Array)
+            matches = expected["code_contains"].select { |phrase| answer.include?(phrase) }
+            match_ratio = matches.size.to_f / expected["code_contains"].size
+            
+            result[:score] = match_ratio
+            result[:reasoning] = "Code snippet matched #{matches.size}/#{expected["code_contains"].size} expected elements"
+            result[:passed] = match_ratio >= 0.5
+            return result
+          end
+          
+          # No match found
+          result[:reasoning] = "No matching criteria found"
+          result
+        end
+        
+        # Extract the answer from the response
+        # @param response [Hash] Response data
+        # @return [String] Extracted answer
+        def self.extract_answer(response)
+          return response["answer"] if response["answer"]
+          return response["text"] if response["text"]
+          return response["content"] if response["content"]
+          return response["output"] if response["output"]
+          return response.to_s
+        end
+      end
+    end
+  end
+end

data/lib/generators/langsmithrb_rails/evals/templates/checks/llm_graded.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module LangsmithrbRails
+  module Evals
+    module Checks
+      # LLM-based grading for evaluating responses
+      class LlmGraded
+        # Check if the response is correct using an LLM
+        # @param input [Hash] Input data
+        # @param response [Hash] Response data
+        # @param expected [Hash] Expected output data
+        # @return [Hash] Evaluation result
+        def self.evaluate(input, response, expected)
+          result = {
+            score: 0.0,
+            reasoning: "",
+            passed: false
+          }
+          
+          # Extract the answer from the response
+          answer = extract_answer(response)
+          expected_answer = extract_answer(expected)
+          
+          # Create the prompt for the LLM
+          prompt = create_grading_prompt(input, answer, expected_answer)
+          
+          # Call the LLM for grading
+          llm_response = call_llm(prompt)
+          
+          # Parse the LLM response
+          parsed_result = parse_llm_response(llm_response)
+          
+          # Update the result with the parsed data
+          result[:score] = parsed_result[:score]
+          result[:reasoning] = parsed_result[:reasoning]
+          result[:passed] = parsed_result[:score] >= 0.7
+          
+          result
+        end
+        
+        # Extract the answer from the response
+        # @param response [Hash] Response data
+        # @return [String] Extracted answer
+        def self.extract_answer(response)
+          return response["answer"] if response["answer"]
+          return response["text"] if response["text"]
+          return response["content"] if response["content"]
+          return response["output"] if response["output"]
+          return response.to_s
+        end
+        
+        # Create a prompt for the LLM to grade the response
+        # @param input [Hash] Input data
+        # @param answer [String] Actual answer
+        # @param expected_answer [String] Expected answer
+        # @return [String] Prompt for the LLM
+        def self.create_grading_prompt(input, answer, expected_answer)
+          <<~PROMPT
+            You are an expert evaluator. Your task is to grade the quality and correctness of a response.
+            
+            Question: #{input["question"]}
+            
+            Expected Answer: #{expected_answer}
+            
+            Actual Response: #{answer}
+            
+            Please evaluate the response based on:
+            1. Correctness: Is the information accurate?
+            2. Completeness: Does it fully address the question?
+            3. Clarity: Is it well-explained and easy to understand?
+            
+            Provide your evaluation in the following format:
+            
+            Score: [a number between 0.0 and 1.0]
+            Reasoning: [your detailed explanation]
+          PROMPT
+        end
+        
+        # Call the LLM for grading
+        # @param prompt [String] Prompt for the LLM
+        # @return [String] LLM response
+        def self.call_llm(prompt)
+          # Check if OpenAI is configured
+          if defined?(OpenAI) && ENV["OPENAI_API_KEY"].present?
+            client = OpenAI::Client.new(access_token: ENV["OPENAI_API_KEY"])
+            response = client.chat(
+              parameters: {
+                model: ENV.fetch("LANGSMITH_EVAL_MODEL", "gpt-3.5-turbo"),
+                messages: [{ role: "user", content: prompt }],
+                temperature: 0.0
+              }
+            )
+            return response.dig("choices", 0, "message", "content")
+          end
+          
+          # Check if Anthropic is configured
+          if defined?(Anthropic) && ENV["ANTHROPIC_API_KEY"].present?
+            client = Anthropic::Client.new(api_key: ENV["ANTHROPIC_API_KEY"])
+            response = client.messages.create(
+              model: ENV.fetch("LANGSMITH_EVAL_MODEL", "claude-2"),
+              max_tokens: 1024,
+              messages: [{ role: "user", content: prompt }]
+            )
+            return response.content.first.text
+          end
+          
+          # Fall back to a simple evaluation
+          "Score: 0.5\nReasoning: Unable to perform LLM-based evaluation. Please configure an LLM provider."
+        end
+        
+        # Parse the LLM response
+        # @param response [String] LLM response
+        # @return [Hash] Parsed result
+        def self.parse_llm_response(response)
+          result = {
+            score: 0.5,
+            reasoning: "Unable to parse LLM response"
+          }
+          
+          # Extract score
+          if response =~ /Score:\s*([\d\.]+)/i
+            result[:score] = $1.to_f
+            # Ensure score is between 0 and 1
+            result[:score] = [0.0, [1.0, result[:score]].min].max
+          end
+          
+          # Extract reasoning
+          if response =~ /Reasoning:\s*(.+)/im
+            result[:reasoning] = $1.strip
+          end
+          
+          result
+        end
+      end
+    end
+  end
+end

data/lib/generators/langsmithrb_rails/evals/templates/datasets/sample.yml

@@ -0,0 +1,60 @@
+# Sample evaluation dataset
+# This file defines a set of inputs and expected outputs for LLM evaluation
+
+name: sample_dataset
+description: A sample dataset for evaluating LLM responses
+version: 1.0.0
+
+# Each item in the dataset represents a test case
+items:
+  - id: question_1
+    input:
+      question: "What is the capital of France?"
+    expected_output:
+      answer: "Paris"
+    metadata:
+      category: geography
+      difficulty: easy
+
+  - id: question_2
+    input:
+      question: "Who wrote 'Pride and Prejudice'?"
+    expected_output:
+      answer: "Jane Austen"
+    metadata:
+      category: literature
+      difficulty: easy
+
+  - id: question_3
+    input:
+      question: "What is the formula for calculating the area of a circle?"
+    expected_output:
+      answer: "A = πr²"
+    metadata:
+      category: mathematics
+      difficulty: medium
+
+  - id: question_4
+    input:
+      question: "Explain how a transformer neural network works."
+    expected_output:
+      answer_contains:
+        - "attention mechanism"
+        - "self-attention"
+        - "encoder"
+        - "decoder"
+    metadata:
+      category: machine_learning
+      difficulty: hard
+
+  - id: question_5
+    input:
+      question: "Write a function to check if a string is a palindrome."
+    expected_output:
+      code_contains:
+        - "function"
+        - "return"
+        - "reverse"
+    metadata:
+      category: programming
+      difficulty: medium