langsmithrb_rails 0.1.0 → 0.3.0

data/lib/generators/langsmithrb_rails/buffer/templates/langsmith.rake

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+namespace :langsmith do
+  desc "Flush the LangSmith buffer"
+  task flush: :environment do
+    puts "Flushing LangSmith buffer..."
+    
+    # Get counts before
+    pending_count = LangsmithRunBuffer.pending.count
+    failed_count = LangsmithRunBuffer.ready_to_retry.count
+    
+    # Process pending traces
+    Langsmith::FlushBufferJob.new.perform(retry_failed: true)
+    
+    # Get counts after
+    pending_after = LangsmithRunBuffer.pending.count
+    failed_after = LangsmithRunBuffer.ready_to_retry.count
+    sent_count = pending_count + failed_count - pending_after - failed_after
+    
+    puts "Sent #{sent_count} traces to LangSmith"
+    puts "#{pending_after} traces still pending"
+    puts "#{failed_after} traces failed"
+  end
+  
+  desc "Replay traces from a file"
+  task :replay, [:file] => :environment do |_t, args|
+    file_path = args[:file]
+    
+    unless file_path && File.exist?(file_path)
+      puts "Error: File not found or not specified"
+      puts "Usage: rake langsmith:replay[path/to/file.json]"
+      exit 1
+    end
+    
+    puts "Replaying traces from #{file_path}..."
+    
+    # Load the file
+    traces = JSON.parse(File.read(file_path))
+    
+    # Create buffer entries
+    traces.each do |trace|
+      LangsmithRunBuffer.create!(
+        name: trace["name"] || "replayed_trace",
+        run_type: trace["run_type"] || "llm",
+        status: "pending",
+        request_id: trace["request_id"],
+        user_ref: trace["user_ref"],
+        run_id: trace["run_id"],
+        parent_run_id: trace["parent_run_id"],
+        started_at: trace["started_at"],
+        ended_at: trace["ended_at"],
+        meta: trace["meta"] || {},
+        payload: trace,
+        error: nil,
+        retry_count: 0
+      )
+    end
+    
+    puts "Created #{traces.size} buffer entries"
+    puts "Run rake langsmith:flush to send them to LangSmith"
+  end
+  
+  desc "Clean up old sent traces"
+  task cleanup: :environment do
+    # Delete traces that were sent more than 7 days ago
+    cutoff_date = 7.days.ago
+    count = LangsmithRunBuffer.sent.where("updated_at < ?", cutoff_date).delete_all
+    
+    puts "Deleted #{count} old traces from the buffer"
+  end
+end

data/lib/generators/langsmithrb_rails/buffer/templates/langsmith_run_buffer.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+# Model for buffering LangSmith traces
+class LangsmithRunBuffer < ApplicationRecord
+  # Validations
+  validates :name, presence: true
+  validates :run_type, presence: true
+  validates :status, presence: true
+  
+  # Scopes
+  scope :pending, -> { where(status: "pending") }
+  scope :failed, -> { where(status: "failed") }
+  scope :sent, -> { where(status: "sent") }
+  scope :ready_to_retry, -> { where(status: "failed").where("retry_count < ?", 5) }
+  
+  # Send the trace to LangSmith
+  # @return [Boolean] Whether the trace was sent successfully
+  def send_to_langsmith
+    client = LangsmithrbRails::Client.new
+    
+    if run_id.present?
+      # This is an update operation
+      response = client.update_run(run_id, payload)
+    else
+      # This is a create operation
+      response = client.create_run(payload)
+      
+      # Store the run ID if available
+      if response&.dig(:status) == 200 && response&.dig(:body, "id").present?
+        update(run_id: response.dig(:body, "id"))
+      end
+    end
+    
+    if response&.dig(:status) == 200
+      update(status: "sent")
+      true
+    else
+      increment!(:retry_count)
+      update(
+        status: "failed", 
+        error: response&.dig(:error) || "Unknown error"
+      )
+      false
+    end
+  rescue => e
+    increment!(:retry_count)
+    update(
+      status: "failed", 
+      error: e.message
+    )
+    false
+  end
+  
+  # Mark the trace as failed
+  # @param error [String] Error message
+  # @return [Boolean] Whether the update was successful
+  def mark_failed(error)
+    increment!(:retry_count)
+    update(
+      status: "failed", 
+      error: error
+    )
+  end
+  
+  # Mark the trace as sent
+  # @return [Boolean] Whether the update was successful
+  def mark_sent
+    update(status: "sent")
+  end
+end

data/lib/generators/langsmithrb_rails/buffer/templates/migration.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+class CreateLangsmithRunBuffers < ActiveRecord::Migration[6.0]
+  def change
+    create_table :langsmith_run_buffers do |t|
+      t.string :name, null: false
+      t.string :run_type, null: false
+      t.string :status, null: false, default: "pending"
+      t.string :request_id
+      t.string :user_ref
+      t.string :run_id
+      t.string :parent_run_id
+      t.datetime :started_at
+      t.datetime :ended_at
+      t.jsonb :meta
+      t.jsonb :payload
+      t.text :error
+      t.integer :retry_count, null: false, default: 0
+      
+      t.timestamps
+    end
+    
+    add_index :langsmith_run_buffers, :status
+    add_index :langsmith_run_buffers, :request_id
+    add_index :langsmith_run_buffers, :run_id
+    add_index :langsmith_run_buffers, :parent_run_id
+  end
+end

data/lib/generators/langsmithrb_rails/ci/ci_generator.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module LangsmithrbRails
+  module Generators
+    # Generator for adding CI integration for LangSmith evaluations
+    class CiGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+      
+      desc "Adds CI integration for LangSmith evaluations"
+      
+      def create_github_workflow
+        empty_directory ".github/workflows"
+        template "langsmith-evals.yml", ".github/workflows/langsmith-evals.yml"
+      end
+      
+      def create_ci_script
+        empty_directory "script/ci"
+        template "langsmith_export_summary.rb", "script/ci/langsmith_export_summary.rb"
+        chmod "script/ci/langsmith_export_summary.rb", 0755
+      end
+      
+      def display_post_install_message
+        say "\n"
+        say "LangSmith CI integration has been added to your Rails application! 🎉", :green
+        say "\n"
+        say "This adds:", :yellow
+        say "  1. GitHub Actions workflow for running evaluations", :yellow
+        say "  2. Script for generating evaluation summaries for PR comments", :yellow
+        say "\n"
+        say "To customize the CI configuration:", :yellow
+        say "  1. Edit .github/workflows/langsmith-evals.yml", :yellow
+        say "  2. Configure the evaluation dataset and target in the workflow", :yellow
+        say "\n"
+      end
+    end
+  end
+end

data/lib/generators/langsmithrb_rails/ci/templates/langsmith-evals.yml

@@ -0,0 +1,85 @@
+name: LangSmith Evaluations
+
+on:
+  pull_request:
+    branches: [ main, master ]
+  workflow_dispatch:
+    inputs:
+      dataset:
+        description: 'Evaluation dataset to use'
+        required: true
+        default: 'sample'
+      target:
+        description: 'Evaluation target to use'
+        required: true
+        default: 'http'
+      experiment_name:
+        description: 'Experiment name'
+        required: false
+        default: ''
+
+jobs:
+  evaluate:
+    name: Run LangSmith Evaluations
+    runs-on: ubuntu-latest
+    
+    env:
+      RAILS_ENV: test
+      LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }}
+      LANGSMITH_PROJECT: ${{ secrets.LANGSMITH_PROJECT || 'ci-evaluations' }}
+      LANGSMITH_EVAL_THRESHOLD: 0.7
+      LANGSMITH_EVAL_TARGET_URL: ${{ secrets.LANGSMITH_EVAL_TARGET_URL || 'http://localhost:3000/api/v1/evaluate' }}
+      
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: '3.2'
+          bundler-cache: true
+      
+      - name: Setup database
+        run: |
+          bundle exec rails db:create
+          bundle exec rails db:schema:load
+      
+      - name: Set experiment name
+        run: |
+          if [ -z "${{ github.event.inputs.experiment_name }}" ]; then
+            echo "EXPERIMENT_NAME=pr_${{ github.event.pull_request.number || github.run_id }}" >> $GITHUB_ENV
+          else
+            echo "EXPERIMENT_NAME=${{ github.event.inputs.experiment_name }}" >> $GITHUB_ENV
+          fi
+      
+      - name: Run evaluations
+        run: |
+          bundle exec rails langsmith:eval[${{ github.event.inputs.dataset || 'sample' }},${{ github.event.inputs.target || 'http' }},${{ env.EXPERIMENT_NAME }}]
+      
+      - name: Generate evaluation summary
+        run: |
+          ruby script/ci/langsmith_export_summary.rb > evaluation_summary.md
+      
+      - name: Upload evaluation results
+        uses: actions/upload-artifact@v3
+        with:
+          name: langsmith-evaluation-results
+          path: |
+            tmp/langsmith/last_eval.json
+            evaluation_summary.md
+      
+      - name: Comment on PR
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            const fs = require('fs');
+            const summary = fs.readFileSync('evaluation_summary.md', 'utf8');
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: summary
+            });

data/lib/generators/langsmithrb_rails/ci/templates/langsmith_export_summary.rb

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+# Path to the evaluation results
+results_path = File.join(Dir.pwd, "tmp/langsmith/last_eval.json")
+
+unless File.exist?(results_path)
+  puts "## LangSmith Evaluation Results"
+  puts
+  puts "⚠️ No evaluation results found. Please run evaluations first."
+  exit 0
+end
+
+# Load the results
+results = JSON.parse(File.read(results_path))
+
+# Generate summary
+puts "## LangSmith Evaluation Results"
+puts
+puts "### Summary"
+puts
+puts "- **Dataset**: #{results['dataset']}"
+puts "- **Target**: #{results['target']}"
+puts "- **Experiment**: #{results['experiment']}"
+puts "- **Timestamp**: #{results['timestamp']}"
+puts "- **Passed**: #{results['summary']['passed']}/#{results['summary']['total']} (#{(results['summary']['passed'].to_f / results['summary']['total'] * 100).round(1)}%)"
+puts "- **Average Score**: #{(results['summary']['avg_score'] * 100).round(1)}%"
+puts
+
+# Determine overall status
+threshold = ENV["LANGSMITH_EVAL_THRESHOLD"] ? ENV["LANGSMITH_EVAL_THRESHOLD"].to_f : 0.7
+threshold_percent = (threshold * 100).round(1)
+
+if results['summary']['avg_score'] >= threshold
+  puts "✅ **PASSED**: Score exceeds threshold of #{threshold_percent}%"
+else
+  puts "❌ **FAILED**: Score below threshold of #{threshold_percent}%"
+end
+
+puts
+
+# Show top 5 items with lowest scores
+puts "### Lowest Scoring Items"
+puts
+
+items_with_scores = results['items'].map do |item|
+  total_score = 0.0
+  item['checks'].each do |_name, check|
+    total_score += check['score']
+  end
+  avg_score = item['checks'].empty? ? 0.0 : (total_score / item['checks'].size)
+  
+  [item['id'], avg_score, item]
+end
+
+items_with_scores.sort_by! { |_, score, _| score }
+
+puts "| ID | Score | Input | Status |"
+puts "| --- | --- | --- | --- |"
+
+items_with_scores.first(5).each do |id, score, item|
+  input_summary = item['input'].map { |k, v| "#{k}: #{v.to_s[0..30]}..." }.join(", ")
+  status = item['passed'] ? "✅" : "❌"
+  puts "| #{id} | #{(score * 100).round(1)}% | #{input_summary} | #{status} |"
+end
+
+puts
+
+# Link to full results
+puts "### Detailed Results"
+puts
+puts "Full results are available as an artifact in the GitHub Actions workflow."
+puts
+puts "To view the results locally, run:"
+puts
+puts "```bash"
+puts "cat tmp/langsmith/last_eval.json | jq"
+puts "```"

data/lib/generators/langsmithrb_rails/demo/demo_generator.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module LangsmithrbRails
+  module Generators
+    # Generator for adding a demo LLM application with LangSmith tracing
+    class DemoGenerator < Rails::Generators::Base
+      source_root File.expand_path("templates", __dir__)
+      
+      desc "Adds a demo LLM application with LangSmith tracing"
+      
+      class_option :provider, type: :string, default: "openai",
+                   desc: "LLM provider to use (openai, anthropic, or mock)"
+      
+      def check_dependencies
+        # Check if the required gems are installed
+        unless options[:provider] == "mock"
+          gem_name = options[:provider] == "anthropic" ? "anthropic" : "ruby-openai"
+          
+          begin
+            require options[:provider] == "anthropic" ? "anthropic" : "openai"
+          rescue LoadError
+            say_status :error, "#{gem_name} gem is required for this generator", :red
+            say "Please add the following to your Gemfile and run bundle install:", :yellow
+            say "gem '#{gem_name}'", :yellow
+            exit 1
+          end
+        end
+      end
+      
+      def create_service
+        template "llm_service.rb", "app/services/llm_service.rb"
+      end
+      
+      def create_controller
+        template "chat_controller.rb", "app/controllers/chat_controller.rb"
+      end
+      
+      def create_views
+        empty_directory "app/views/chat"
+        template "index.html.erb", "app/views/chat/index.html.erb"
+      end
+      
+      def create_model
+        template "chat_message.rb", "app/models/chat_message.rb"
+      end
+      
+      def create_migration
+        migration_template "create_chat_messages.rb", "db/migrate/create_chat_messages.rb"
+      end
+      
+      def update_routes
+        route "resources :chat, only: [:index, :create]"
+      end
+      
+      def create_javascript
+        empty_directory "app/javascript/controllers"
+        template "chat_controller.js", "app/javascript/controllers/chat_controller.js"
+      end
+      
+      def display_post_install_message
+        say "\n"
+        say "LangSmith demo application has been added to your Rails application! 🎉", :green
+        say "\n"
+        say "This adds:", :yellow
+        say "  1. LLM service with LangSmith tracing", :yellow
+        say "  2. Chat controller and views", :yellow
+        say "  3. Chat message model and migration", :yellow
+        say "\n"
+        say "To set up the demo:", :yellow
+        say "  1. Run the migration: bin/rails db:migrate", :yellow
+        say "  2. Configure your LLM provider API key in your environment", :yellow
+        say "  3. Start your Rails server: bin/rails server", :yellow
+        say "  4. Visit http://localhost:3000/chat", :yellow
+        say "\n"
+        say "For OpenAI, set OPENAI_API_KEY in your environment", :yellow
+        say "For Anthropic, set ANTHROPIC_API_KEY in your environment", :yellow
+        say "\n"
+      end
+    end
+  end
+end

data/lib/generators/langsmithrb_rails/demo/templates/chat_controller.js

@@ -0,0 +1,88 @@
+// Chat controller for Stimulus.js
+import { Controller } from "@hotwired/stimulus"
+
+export default class extends Controller {
+  static targets = ["input", "submitButton"]
+  
+  connect() {
+    // Scroll to bottom of messages on load
+    this.scrollToBottom()
+    
+    // Setup keyboard shortcuts
+    this.inputTarget.addEventListener("keydown", this.handleKeyDown.bind(this))
+  }
+  
+  // Handle form submission
+  async submit(event) {
+    event.preventDefault()
+    
+    const message = this.inputTarget.value.trim()
+    if (!message) return
+    
+    // Disable input and button during submission
+    this.inputTarget.disabled = true
+    this.submitButton.disabled = true
+    this.submitButton.textContent = "Sending..."
+    
+    try {
+      const form = event.target
+      const formData = new FormData(form)
+      
+      const response = await fetch(form.action, {
+        method: form.method,
+        body: formData,
+        headers: {
+          "Accept": "application/json",
+          "X-Requested-With": "XMLHttpRequest"
+        }
+      })
+      
+      if (response.ok) {
+        const data = await response.json()
+        
+        // Add messages to the chat
+        const messagesContainer = document.getElementById("chat-messages")
+        messagesContainer.insertAdjacentHTML("beforeend", data.user_message)
+        
+        // Clear input
+        this.inputTarget.value = ""
+        
+        // Scroll to bottom
+        this.scrollToBottom()
+        
+        // Add assistant message with a slight delay for better UX
+        setTimeout(() => {
+          messagesContainer.insertAdjacentHTML("beforeend", data.assistant_message)
+          this.scrollToBottom()
+        }, 500)
+      } else {
+        const errorData = await response.json()
+        alert(`Error: ${errorData.error || "Something went wrong"}`)
+      }
+    } catch (error) {
+      console.error("Error submitting message:", error)
+      alert("Failed to send message. Please try again.")
+    } finally {
+      // Re-enable input and button
+      this.inputTarget.disabled = false
+      this.submitButton.disabled = false
+      this.submitButton.textContent = "Send"
+      this.inputTarget.focus()
+    }
+  }
+  
+  // Handle keyboard shortcuts
+  handleKeyDown(event) {
+    // Submit on Enter (without Shift)
+    if (event.key === "Enter" && !event.shiftKey) {
+      event.preventDefault()
+      this.submitButton.click()
+    }
+  }
+  
+  // Scroll to the bottom of the messages container
+  scrollToBottom() {
+    const messagesContainer = document.getElementById("chat-messages")
+    messagesContainer.scrollTop = messagesContainer.scrollHeight
+  }
+}

data/lib/generators/langsmithrb_rails/demo/templates/chat_controller.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# Controller for the chat demo
+class ChatController < ApplicationController
+  include LangsmithrbRails::TracedController
+  
+  # GET /chat
+  def index
+    @messages = ChatMessage.order(created_at: :asc).last(10)
+    @message = ChatMessage.new
+  end
+  
+  # POST /chat
+  def create
+    # Create trace for the entire request
+    langsmith_trace("chat_request", run_type: "chain") do |run|
+      @message = ChatMessage.new(message_params)
+      @message.is_user = true
+      
+      if @message.save
+        # Record the user message in the trace
+        run.inputs = { user_message: @message.content }
+        
+        # Generate a response
+        llm_service = LlmService.new
+        context = ChatMessage.order(created_at: :asc).last(5).map do |msg|
+          { is_user: msg.is_user, content: msg.content }
+        end
+        
+        response = llm_service.generate(@message.content, context)
+        
+        # Create the assistant message
+        @assistant_message = ChatMessage.create!(
+          content: response,
+          is_user: false
+        )
+        
+        # Record the assistant response in the trace
+        run.outputs = { assistant_message: @assistant_message.content }
+        
+        # Respond with both messages for AJAX updates
+        render json: {
+          user_message: render_to_string(partial: "message", locals: { message: @message }),
+          assistant_message: render_to_string(partial: "message", locals: { message: @assistant_message })
+        }
+      else
+        render json: { error: @message.errors.full_messages.join(", ") }, status: :unprocessable_entity
+      end
+    end
+  end
+  
+  private
+  
+  # Only allow a list of trusted parameters through
+  def message_params
+    params.require(:chat_message).permit(:content)
+  end
+end

data/lib/generators/langsmithrb_rails/demo/templates/chat_message.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# Model for storing chat messages
+class ChatMessage < ApplicationRecord
+  # Validations
+  validates :content, presence: true
+  
+  # Scopes
+  scope :user_messages, -> { where(is_user: true) }
+  scope :assistant_messages, -> { where(is_user: false) }
+  scope :recent, -> { order(created_at: :desc) }
+  
+  # Get the conversation history as an array of messages
+  # @param limit [Integer] Maximum number of messages to return
+  # @return [Array<Hash>] Array of messages with role and content
+  def self.conversation_history(limit = 10)
+    recent.limit(limit).map do |message|
+      {
+        role: message.is_user? ? "user" : "assistant",
+        content: message.content
+      }
+    end.reverse
+  end
+end

data/lib/generators/langsmithrb_rails/demo/templates/create_chat_messages.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# Migration for creating the chat_messages table
+class CreateChatMessages < ActiveRecord::Migration[7.0]
+  def change
+    create_table :chat_messages do |t|
+      t.text :content, null: false
+      t.boolean :is_user, default: true
+      t.string :metadata
+      t.string :langsmith_run_id
+
+      t.timestamps
+    end
+    
+    add_index :chat_messages, :is_user
+    add_index :chat_messages, :langsmith_run_id
+    add_index :chat_messages, :created_at
+  end
+end