langsmithrb_rails 0.1.0 → 0.1.1

data/lib/generators/langsmithrb_rails/ci/templates/langsmith-evals.yml

@@ -0,0 +1,85 @@
+name: LangSmith Evaluations
+
+on:
+  pull_request:
+    branches: [ main, master ]
+  workflow_dispatch:
+    inputs:
+      dataset:
+        description: 'Evaluation dataset to use'
+        required: true
+        default: 'sample'
+      target:
+        description: 'Evaluation target to use'
+        required: true
+        default: 'http'
+      experiment_name:
+        description: 'Experiment name'
+        required: false
+        default: ''
+
+jobs:
+  evaluate:
+    name: Run LangSmith Evaluations
+    runs-on: ubuntu-latest
+    
+    env:
+      RAILS_ENV: test
+      LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }}
+      LANGSMITH_PROJECT: ${{ secrets.LANGSMITH_PROJECT || 'ci-evaluations' }}
+      LANGSMITH_EVAL_THRESHOLD: 0.7
+      LANGSMITH_EVAL_TARGET_URL: ${{ secrets.LANGSMITH_EVAL_TARGET_URL || 'http://localhost:3000/api/v1/evaluate' }}
+      
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.2'
+          bundler-cache: true
+      
+      - name: Setup database
+        run: |
+          bundle exec rails db:create
+          bundle exec rails db:schema:load
+      
+      - name: Set experiment name
+        run: |
+          if [ -z "${{ github.event.inputs.experiment_name }}" ]; then
+            echo "EXPERIMENT_NAME=pr_${{ github.event.pull_request.number || github.run_id }}" >> $GITHUB_ENV
+          else
+            echo "EXPERIMENT_NAME=${{ github.event.inputs.experiment_name }}" >> $GITHUB_ENV
+          fi
+      
+      - name: Run evaluations
+        run: |
+          bundle exec rails langsmith:eval[${{ github.event.inputs.dataset || 'sample' }},${{ github.event.inputs.target || 'http' }},${{ env.EXPERIMENT_NAME }}]
+      
+      - name: Generate evaluation summary
+        run: |
+          ruby script/ci/langsmith_export_summary.rb > evaluation_summary.md
+      
+      - name: Upload evaluation results
+        uses: actions/upload-artifact@v3
+        with:
+          name: langsmith-evaluation-results
+          path: |
+            tmp/langsmith/last_eval.json
+            evaluation_summary.md
+      
+      - name: Comment on PR
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            const summary = fs.readFileSync('evaluation_summary.md', 'utf8');
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: summary
+            });

data/lib/generators/langsmithrb_rails/ci/templates/langsmith_export_summary.rb

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+# Path to the evaluation results
+results_path = File.join(Dir.pwd, "tmp/langsmith/last_eval.json")
+
+unless File.exist?(results_path)
+  puts "## LangSmith Evaluation Results"
+  puts
+  puts "⚠️ No evaluation results found. Please run evaluations first."
+  exit 0
+end
+
+# Load the results
+results = JSON.parse(File.read(results_path))
+
+# Generate summary
+puts "## LangSmith Evaluation Results"
+puts
+puts "### Summary"
+puts
+puts "- **Dataset**: #{results['dataset']}"
+puts "- **Target**: #{results['target']}"
+puts "- **Experiment**: #{results['experiment']}"
+puts "- **Timestamp**: #{results['timestamp']}"
+puts "- **Passed**: #{results['summary']['passed']}/#{results['summary']['total']} (#{(results['summary']['passed'].to_f / results['summary']['total'] * 100).round(1)}%)"
+puts "- **Average Score**: #{(results['summary']['avg_score'] * 100).round(1)}%"
+puts
+
+# Determine overall status
+threshold = ENV["LANGSMITH_EVAL_THRESHOLD"] ? ENV["LANGSMITH_EVAL_THRESHOLD"].to_f : 0.7
+threshold_percent = (threshold * 100).round(1)
+
+if results['summary']['avg_score'] >= threshold
+  puts "✅ **PASSED**: Score exceeds threshold of #{threshold_percent}%"
+else
+  puts "❌ **FAILED**: Score below threshold of #{threshold_percent}%"
+end
+
+puts
+
+# Show top 5 items with lowest scores
+puts "### Lowest Scoring Items"
+puts
+
+items_with_scores = results['items'].map do |item|
+  total_score = 0.0
+  item['checks'].each do |_name, check|
+    total_score += check['score']
+  end
+  avg_score = item['checks'].empty? ? 0.0 : (total_score / item['checks'].size)
+  
+  [item['id'], avg_score, item]
+end
+
+items_with_scores.sort_by! { |_, score, _| score }
+
+puts "| ID | Score | Input | Status |"
+puts "| --- | --- | --- | --- |"
+
+items_with_scores.first(5).each do |id, score, item|
+  input_summary = item['input'].map { |k, v| "#{k}: #{v.to_s[0..30]}..." }.join(", ")
+  status = item['passed'] ? "✅" : "❌"
+  puts "| #{id} | #{(score * 100).round(1)}% | #{input_summary} | #{status} |"
+end
+
+puts
+
+# Link to full results
+puts "### Detailed Results"
+puts
+puts "Full results are available as an artifact in the GitHub Actions workflow."
+puts
+puts "To view the results locally, run:"
+puts
+puts "```bash"
+puts "cat tmp/langsmith/last_eval.json | jq"
+puts "```"

data/lib/generators/langsmithrb_rails/demo/demo_generator.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module LangsmithrbRails
+  module Generators
+    # Generator for adding a demo LLM application with LangSmith tracing
+    class DemoGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+      
+      desc "Adds a demo LLM application with LangSmith tracing"
+      
+      class_option :provider, type: :string, default: "openai",
+                   desc: "LLM provider to use (openai, anthropic, or mock)"
+      
+      def check_dependencies
+        # Check if the required gems are installed
+        unless options[:provider] == "mock"
+          gem_name = options[:provider] == "anthropic" ? "anthropic" : "ruby-openai"
+          
+          begin
+            require options[:provider] == "anthropic" ? "anthropic" : "openai"
+          rescue LoadError
+            say_status :error, "#{gem_name} gem is required for this generator", :red
+            say "Please add the following to your Gemfile and run bundle install:", :yellow
+            say "gem '#{gem_name}'", :yellow
+            exit 1
+          end
+        end
+      end
+      
+      def create_service
+        template "llm_service.rb", "app/services/llm_service.rb"
+      end
+      
+      def create_controller
+        template "chat_controller.rb", "app/controllers/chat_controller.rb"
+      end
+      
+      def create_views
+        empty_directory "app/views/chat"
+        template "index.html.erb", "app/views/chat/index.html.erb"
+      end
+      
+      def create_model
+        template "chat_message.rb", "app/models/chat_message.rb"
+      end
+      
+      def create_migration
+        migration_template "create_chat_messages.rb", "db/migrate/create_chat_messages.rb"
+      end
+      
+      def update_routes
+        route "resources :chat, only: [:index, :create]"
+      end
+      
+      def create_javascript
+        empty_directory "app/javascript/controllers"
+        template "chat_controller.js", "app/javascript/controllers/chat_controller.js"
+      end
+      
+      def display_post_install_message
+        say "\n"
+        say "LangSmith demo application has been added to your Rails application! 🎉", :green
+        say "\n"
+        say "This adds:", :yellow
+        say "  1. LLM service with LangSmith tracing", :yellow
+        say "  2. Chat controller and views", :yellow
+        say "  3. Chat message model and migration", :yellow
+        say "\n"
+        say "To set up the demo:", :yellow
+        say "  1. Run the migration: bin/rails db:migrate", :yellow
+        say "  2. Configure your LLM provider API key in your environment", :yellow
+        say "  3. Start your Rails server: bin/rails server", :yellow
+        say "  4. Visit http://localhost:3000/chat", :yellow
+        say "\n"
+        say "For OpenAI, set OPENAI_API_KEY in your environment", :yellow
+        say "For Anthropic, set ANTHROPIC_API_KEY in your environment", :yellow
+        say "\n"
+      end
+    end
+  end
+end

data/lib/generators/langsmithrb_rails/demo/templates/chat_controller.js

@@ -0,0 +1,88 @@
+// Chat controller for Stimulus.js
+import { Controller } from "@hotwired/stimulus"
+
+export default class extends Controller {
+  static targets = ["input", "submitButton"]
+  
+  connect() {
+    // Scroll to bottom of messages on load
+    this.scrollToBottom()
+    
+    // Setup keyboard shortcuts
+    this.inputTarget.addEventListener("keydown", this.handleKeyDown.bind(this))
+  }
+  
+  // Handle form submission
+  async submit(event) {
+    event.preventDefault()
+    
+    const message = this.inputTarget.value.trim()
+    if (!message) return
+    
+    // Disable input and button during submission
+    this.inputTarget.disabled = true
+    this.submitButton.disabled = true
+    this.submitButton.textContent = "Sending..."
+    
+    try {
+      const form = event.target
+      const formData = new FormData(form)
+      
+      const response = await fetch(form.action, {
+        method: form.method,
+        body: formData,
+        headers: {
+          "Accept": "application/json",
+          "X-Requested-With": "XMLHttpRequest"
+        }
+      })
+      
+      if (response.ok) {
+        const data = await response.json()
+        
+        // Add messages to the chat
+        const messagesContainer = document.getElementById("chat-messages")
+        messagesContainer.insertAdjacentHTML("beforeend", data.user_message)
+        
+        // Clear input
+        this.inputTarget.value = ""
+        
+        // Scroll to bottom
+        this.scrollToBottom()
+        
+        // Add assistant message with a slight delay for better UX
+        setTimeout(() => {
+          messagesContainer.insertAdjacentHTML("beforeend", data.assistant_message)
+          this.scrollToBottom()
+        }, 500)
+      } else {
+        const errorData = await response.json()
+        alert(`Error: ${errorData.error || "Something went wrong"}`)
+      }
+    } catch (error) {
+      console.error("Error submitting message:", error)
+      alert("Failed to send message. Please try again.")
+    } finally {
+      // Re-enable input and button
+      this.inputTarget.disabled = false
+      this.submitButton.disabled = false
+      this.submitButton.textContent = "Send"
+      this.inputTarget.focus()
+    }
+  }
+  
+  // Handle keyboard shortcuts
+  handleKeyDown(event) {
+    // Submit on Enter (without Shift)
+    if (event.key === "Enter" && !event.shiftKey) {
+      event.preventDefault()
+      this.submitButton.click()
+    }
+  }
+  
+  // Scroll to the bottom of the messages container
+  scrollToBottom() {
+    const messagesContainer = document.getElementById("chat-messages")
+    messagesContainer.scrollTop = messagesContainer.scrollHeight
+  }
+}

data/lib/generators/langsmithrb_rails/demo/templates/chat_controller.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# Controller for the chat demo
+class ChatController < ApplicationController
+  include LangsmithrbRails::TracedController
+  
+  # GET /chat
+  def index
+    @messages = ChatMessage.order(created_at: :asc).last(10)
+    @message = ChatMessage.new
+  end
+  
+  # POST /chat
+  def create
+    # Create trace for the entire request
+    langsmith_trace("chat_request", run_type: "chain") do |run|
+      @message = ChatMessage.new(message_params)
+      @message.is_user = true
+      
+      if @message.save
+        # Record the user message in the trace
+        run.inputs = { user_message: @message.content }
+        
+        # Generate a response
+        llm_service = LlmService.new
+        context = ChatMessage.order(created_at: :asc).last(5).map do |msg|
+          { is_user: msg.is_user, content: msg.content }
+        end
+        
+        response = llm_service.generate(@message.content, context)
+        
+        # Create the assistant message
+        @assistant_message = ChatMessage.create!(
+          content: response,
+          is_user: false
+        )
+        
+        # Record the assistant response in the trace
+        run.outputs = { assistant_message: @assistant_message.content }
+        
+        # Respond with both messages for AJAX updates
+        render json: {
+          user_message: render_to_string(partial: "message", locals: { message: @message }),
+          assistant_message: render_to_string(partial: "message", locals: { message: @assistant_message })
+        }
+      else
+        render json: { error: @message.errors.full_messages.join(", ") }, status: :unprocessable_entity
+      end
+    end
+  end
+  
+  private
+  
+  # Only allow a list of trusted parameters through
+  def message_params
+    params.require(:chat_message).permit(:content)
+  end
+end

data/lib/generators/langsmithrb_rails/demo/templates/chat_message.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# Model for storing chat messages
+class ChatMessage < ApplicationRecord
+  # Validations
+  validates :content, presence: true
+  
+  # Scopes
+  scope :user_messages, -> { where(is_user: true) }
+  scope :assistant_messages, -> { where(is_user: false) }
+  scope :recent, -> { order(created_at: :desc) }
+  
+  # Get the conversation history as an array of messages
+  # @param limit [Integer] Maximum number of messages to return
+  # @return [Array<Hash>] Array of messages with role and content
+  def self.conversation_history(limit = 10)
+    recent.limit(limit).map do |message|
+      {
+        role: message.is_user? ? "user" : "assistant",
+        content: message.content
+      }
+    end.reverse
+  end
+end

data/lib/generators/langsmithrb_rails/demo/templates/create_chat_messages.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# Migration for creating the chat_messages table
+class CreateChatMessages < ActiveRecord::Migration[7.0]
+  def change
+    create_table :chat_messages do |t|
+      t.text :content, null: false
+      t.boolean :is_user, default: true
+      t.string :metadata
+      t.string :langsmith_run_id
+
+      t.timestamps
+    end
+    
+    add_index :chat_messages, :is_user
+    add_index :chat_messages, :langsmith_run_id
+    add_index :chat_messages, :created_at
+  end
+end

data/lib/generators/langsmithrb_rails/demo/templates/index.html.erb

@@ -0,0 +1,180 @@
+<%# Chat demo index page %>
+<div class="chat-container" data-controller="chat">
+  <div class="chat-header">
+    <h1>LangSmith Rails Demo</h1>
+    <p class="subtitle">A simple chat interface with LangSmith tracing</p>
+  </div>
+  
+  <div class="chat-messages" id="chat-messages">
+    <% if @messages.empty? %>
+      <div class="welcome-message">
+        <h2>Welcome to the LangSmith Rails Demo!</h2>
+        <p>This is a simple chat interface that demonstrates LangSmith tracing in a Rails application.</p>
+        <p>Every message you send will be traced with LangSmith, allowing you to monitor and debug your LLM interactions.</p>
+        <p>Try asking a question below to get started!</p>
+      </div>
+    <% else %>
+      <% @messages.each do |message| %>
+        <%= render partial: "message", locals: { message: message } %>
+      <% end %>
+    <% end %>
+  </div>
+  
+  <div class="chat-form">
+    <%= form_with(model: @message, url: chat_index_path, data: { action: "submit->chat#submit" }) do |form| %>
+      <div class="input-group">
+        <%= form.text_area :content, placeholder: "Type your message...", class: "chat-input", data: { chat_target: "input" } %>
+        <button type="submit" class="send-button" data-chat-target="submitButton">
+          Send
+        </button>
+      </div>
+    <% end %>
+  </div>
+  
+  <div class="chat-footer">
+    <p>
+      Powered by LangSmith Rails
+      <% if ENV["LANGSMITH_PROJECT"].present? %>
+        | Project: <%= ENV["LANGSMITH_PROJECT"] %>
+      <% end %>
+    </p>
+  </div>
+</div>
+
+<%# Partial for rendering a message %>
+<% content_for :partials do %>
+  <script type="text/html" id="message-template">
+    <div class="message <%= message.is_user? ? 'user-message' : 'assistant-message' %>">
+      <div class="message-content">
+        <%= message.content %>
+      </div>
+      <div class="message-meta">
+        <span class="message-time"><%= message.created_at.strftime("%H:%M") %></span>
+        <span class="message-role"><%= message.is_user? ? "You" : "Assistant" %></span>
+      </div>
+    </div>
+  </script>
+<% end %>
+
+<style>
+  .chat-container {
+    max-width: 800px;
+    margin: 0 auto;
+    padding: 20px;
+    font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu, Cantarell, "Open Sans", "Helvetica Neue", sans-serif;
+  }
+  
+  .chat-header {
+    text-align: center;
+    margin-bottom: 20px;
+    padding-bottom: 10px;
+    border-bottom: 1px solid #eaeaea;
+  }
+  
+  .chat-header h1 {
+    margin-bottom: 5px;
+    color: #333;
+  }
+  
+  .subtitle {
+    color: #666;
+    margin-top: 0;
+  }
+  
+  .chat-messages {
+    height: 500px;
+    overflow-y: auto;
+    padding: 10px;
+    background-color: #f9f9f9;
+    border-radius: 8px;
+    margin-bottom: 20px;
+  }
+  
+  .welcome-message {
+    text-align: center;
+    padding: 40px 20px;
+    color: #666;
+  }
+  
+  .welcome-message h2 {
+    color: #333;
+    margin-bottom: 15px;
+  }
+  
+  .message {
+    margin-bottom: 15px;
+    padding: 10px 15px;
+    border-radius: 8px;
+    max-width: 80%;
+    position: relative;
+  }
+  
+  .user-message {
+    background-color: #e1f5fe;
+    margin-left: auto;
+    border-bottom-right-radius: 0;
+  }
+  
+  .assistant-message {
+    background-color: #f0f0f0;
+    margin-right: auto;
+    border-bottom-left-radius: 0;
+  }
+  
+  .message-content {
+    white-space: pre-wrap;
+  }
+  
+  .message-meta {
+    font-size: 0.8em;
+    color: #999;
+    margin-top: 5px;
+    text-align: right;
+  }
+  
+  .chat-form {
+    margin-bottom: 20px;
+  }
+  
+  .input-group {
+    display: flex;
+    gap: 10px;
+  }
+  
+  .chat-input {
+    flex-grow: 1;
+    padding: 10px;
+    border: 1px solid #ddd;
+    border-radius: 4px;
+    resize: none;
+    height: 60px;
+    font-family: inherit;
+  }
+  
+  .send-button {
+    padding: 10px 20px;
+    background-color: #2196f3;
+    color: white;
+    border: none;
+    border-radius: 4px;
+    cursor: pointer;
+    transition: background-color 0.2s;
+  }
+  
+  .send-button:hover {
+    background-color: #0b7dda;
+  }
+  
+  .send-button:disabled {
+    background-color: #cccccc;
+    cursor: not-allowed;
+  }
+  
+  .chat-footer {
+    text-align: center;
+    color: #999;
+    font-size: 0.8em;
+    padding-top: 10px;
+    border-top: 1px solid #eaeaea;
+  }
+</style>