llm_logs 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 25fbd9f60530d337d2cfaa0bc4cec00afc6a3c1c802c771d4109460fac64dfea
+  data.tar.gz: 5e406b0764ebe2575e5b19da0b5d73934f46e63aff3f1572f385aa48d0839b02
+SHA512:
+  metadata.gz: 3ad9d74c2d111ad34e4d132d9fc6ab1c818f9c59bc7143bfee9c68a69e4f89349e48c1710412ac54e004aec8dca544abe1fa99fc9f40c31a82c524a452f1c73b
+  data.tar.gz: '0854b4994c2156b431f71fbe08b7f36032e53452962c88ad3467c6fed3c165b37adb47111196c2268d2696f827499ad1fa35c3e69695ea04530f05a98ed0edca'

data/README.md

@@ -0,0 +1,148 @@
+# LlmLogs
+
+Mountable Rails engine for LLM call tracing and prompt management. Auto-instruments [ruby_llm](https://github.com/crmne/ruby_llm) to capture every LLM call as a span within a trace, and provides a web UI for browsing logs and editing prompts.
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem "llm_logs", github: "tonic20/llm_logs"
+```
+
+Run the install generator:
+
+```sh
+bin/rails generate llm_logs:install
+bin/rails db:migrate
+```
+
+Or manually:
+
+```ruby
+# config/routes.rb
+mount LlmLogs::Engine, at: "/llm_logs"
+```
+
+```ruby
+# config/initializers/llm_logs.rb
+LlmLogs.setup do |config|
+  config.enabled = true
+  config.auto_instrument = true
+end
+```
+
+## Tracing
+
+Wrap operations in a trace block. All `ruby_llm` calls inside become child spans automatically.
+
+```ruby
+LlmLogs.trace("strategy_analysis", metadata: { strategy_id: 42 }) do
+  chat = RubyLLM.chat(model: "anthropic/claude-sonnet-4")
+  chat.ask("Analyze this strategy...")
+end
+```
+
+Nested traces are supported:
+
+```ruby
+LlmLogs.trace("full_pipeline") do
+  chat.ask("Step 1...")
+
+  LlmLogs.trace("risk_assessment") do
+    chat.ask("What are the risks?")
+  end
+end
+```
+
+LLM calls made outside an explicit trace are auto-wrapped in one.
+
+### What Gets Captured
+
+Each span records:
+
+- Model and provider
+- Input messages and output content
+- Input, output, and cached token counts
+- Duration in milliseconds
+- Error messages (on failure)
+- Custom metadata
+
+Tool calls are captured as child spans with tool name and arguments.
+
+## Prompt Management
+
+Create prompts with [Mustache](https://mustache.github.io/) templates and auto-versioning.
+
+### Create a Prompt
+
+```ruby
+prompt = LlmLogs::Prompt.create!(
+  slug: "strategy-analysis",
+  name: "Strategy Analysis"
+)
+
+prompt.update_content!(
+  messages: [
+    { "role" => "system", "content" => "You analyze trading strategies for {{app_name}}." },
+    { "role" => "user", "content" => "Analyze {{strategy_name}} on the {{timeframe}} timeframe." }
+  ],
+  model: "claude-sonnet-4",
+  model_params: { "temperature" => 0.3, "max_tokens" => 2048 }
+)
+```
+
+### Load and Render
+
+```ruby
+prompt = LlmLogs::Prompt.load("strategy-analysis")
+params = prompt.build(
+  app_name: "Tradebot",
+  strategy_name: "Momentum Alpha",
+  timeframe: "4h"
+)
+# => { model: "claude-sonnet-4", messages: [...], temperature: 0.3, max_tokens: 2048 }
+```
+
+### Versioning
+
+Every save creates a new version. Previous versions are never modified.
+
+```ruby
+prompt.update_content!(
+  messages: [{ "role" => "user", "content" => "Updated prompt" }],
+  changelog: "Simplified the prompt"
+)
+
+prompt.current_version       # latest
+prompt.version(1)            # specific version
+prompt.rollback_to!(1)       # creates new version from v1 content
+```
+
+## Web UI
+
+Browse traces and manage prompts at `/llm_logs`.
+
+**Traces** — list with filtering by status, drill into hierarchical span trees with collapsible input/output.
+
+**Prompts** — CRUD with Mustache template editor, model configuration, and version history.
+
+## Configuration
+
+```ruby
+LlmLogs.setup do |config|
+  config.enabled = true               # master switch for logging
+  config.auto_instrument = true       # auto-prepend on RubyLLM::Chat
+  config.retention_days = 30          # for future cleanup job
+end
+```
+
+## Requirements
+
+- Rails 8.0+
+- Ruby 3.3+
+- PostgreSQL
+
+## License
+
+MIT

data/Rakefile

@@ -0,0 +1,11 @@
+require "bundler/setup"
+
+APP_RAKEFILE = File.expand_path("spec/dummy/Rakefile", __dir__)
+load "rails/tasks/engine.rake"
+
+require "bundler/gem_tasks"
+
+require "rspec/core/rake_task"
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/app/controllers/llm_logs/application_controller.rb

@@ -0,0 +1,5 @@
+module LlmLogs
+  class ApplicationController < ::ActionController::Base
+    layout "llm_logs/application"
+  end
+end

data/app/controllers/llm_logs/prompt_versions_controller.rb

@@ -0,0 +1,66 @@
+require "diffy"
+
+module LlmLogs
+  class PromptVersionsController < ApplicationController
+    before_action :set_prompt
+
+    def index
+      @versions = @prompt.versions.order(version_number: :desc)
+    end
+
+    def show
+      @version = @prompt.versions.find(params[:id])
+    end
+
+    def restore
+      version = @prompt.versions.find(params[:id])
+      @prompt.rollback_to!(version.version_number)
+      redirect_to prompt_path(@prompt), notice: "Restored to version #{version.version_number}."
+    end
+
+    def destroy
+      version = @prompt.versions.find(params[:id])
+
+      if version == @prompt.current_version
+        redirect_to prompt_versions_path(@prompt), alert: "Cannot delete the current active version."
+        return
+      end
+
+      version.destroy!
+      redirect_to prompt_versions_path(@prompt), notice: "Version #{version.version_number} deleted."
+    end
+
+    def compare
+
+      if params[:a].blank? || params[:b].blank? || params[:a] == params[:b]
+        redirect_to prompt_versions_path(@prompt), alert: "Select two different versions to compare."
+        return
+      end
+
+      @version_a = @prompt.versions.find_by(version_number: params[:a])
+      @version_b = @prompt.versions.find_by(version_number: params[:b])
+
+      unless @version_a && @version_b
+        redirect_to prompt_versions_path(@prompt), alert: "Version not found."
+        return
+      end
+
+      max_messages = [@version_a.messages.size, @version_b.messages.size].max
+      @diffs = (0...max_messages).map do |i|
+        msg_a = @version_a.messages[i]
+        msg_b = @version_b.messages[i]
+        role = (msg_a || msg_b)["role"]
+        content_a = (msg_a&.dig("content") || "").gsub("\r\n", "\n")
+        content_b = (msg_b&.dig("content") || "").gsub("\r\n", "\n")
+        diff_html = Diffy::SplitDiff.new(content_a, content_b, format: :html_simple)
+        { role: role, left: diff_html.left, right: diff_html.right }
+      end
+    end
+
+    private
+
+    def set_prompt
+      @prompt = Prompt.find(params[:prompt_id])
+    end
+  end
+end

data/app/controllers/llm_logs/prompts_controller.rb

@@ -0,0 +1,91 @@
+module LlmLogs
+  class PromptsController < ApplicationController
+    def index
+      @prompts = Prompt.order(:name).includes(:versions).page(params[:page]).per(25)
+    end
+
+    def show
+      @prompt = Prompt.find(params[:id])
+      @current_version = @prompt.current_version
+      @versions = @prompt.versions.order(version_number: :desc).limit(5)
+    end
+
+    def new
+      @prompt = Prompt.new
+    end
+
+    def create
+      @prompt = Prompt.new(prompt_params)
+
+      if @prompt.save
+        if version_params[:messages].present?
+          @prompt.update_content!(**version_params)
+        end
+        redirect_to prompt_path(@prompt), notice: "Prompt created."
+      else
+        render :new, status: :unprocessable_entity
+      end
+    end
+
+    def edit
+      @prompt = Prompt.find(params[:id])
+      @current_version = @prompt.current_version
+    end
+
+    def update
+      @prompt = Prompt.find(params[:id])
+
+      if @prompt.update(prompt_params)
+        if version_params[:messages].present?
+          @prompt.update_content!(**version_params)
+        end
+        redirect_to prompt_path(@prompt), notice: "Prompt updated."
+      else
+        render :edit, status: :unprocessable_entity
+      end
+    end
+
+    def destroy
+      @prompt = Prompt.find(params[:id])
+      @prompt.destroy
+      redirect_to prompts_path, notice: "Prompt deleted."
+    end
+
+    private
+
+    def prompt_params
+      params.require(:prompt).permit(:slug, :name, :description)
+    end
+
+    def version_params
+      raw = params.require(:prompt).permit(:model, :changelog, model_params: {})
+      messages = parse_messages
+      {
+        messages: messages,
+        model: raw[:model],
+        model_params: coerce_model_params(raw[:model_params]&.to_h || {}),
+        changelog: raw[:changelog]
+      }.compact_blank
+    end
+
+    def coerce_model_params(params_hash)
+      params_hash.each_with_object({}) do |(key, value), result|
+        next if value.blank?
+
+        result[key] = case value
+        when /\A\d+\z/ then value.to_i
+        when /\A\d+\.\d+\z/ then value.to_f
+        else value
+        end
+      end
+    end
+
+    def parse_messages
+      return [] unless params[:prompt][:messages].present?
+
+      params[:prompt][:messages].values.map do |msg|
+        { "role" => msg[:role], "content" => msg[:content] }
+      end.reject { |m| m["content"].blank? }
+    end
+  end
+end

data/app/controllers/llm_logs/spans_controller.rb

@@ -0,0 +1,8 @@
+module LlmLogs
+  class SpansController < ApplicationController
+    def show
+      @trace = Trace.find(params[:trace_id])
+      @span = @trace.spans.find(params[:id])
+    end
+  end
+end

data/app/controllers/llm_logs/traces_controller.rb

@@ -0,0 +1,19 @@
+module LlmLogs
+  class TracesController < ApplicationController
+    def index
+      @traces = Trace.recent
+      @traces = @traces.by_status(params[:status]) if params[:status].present?
+      if params[:prompt_version_id].present?
+        @traces = @traces.where(prompt_version_id: params[:prompt_version_id])
+        @filter_version = PromptVersion.find_by(id: params[:prompt_version_id])
+      end
+      @traces = @traces.page(params[:page]).per(50)
+    end
+
+    def show
+      @trace = Trace.includes(prompt_version: :prompt).find(params[:id])
+      @root_spans = @trace.root_spans
+    end
+
+  end
+end

data/app/helpers/llm_logs/formatting_helper.rb

@@ -0,0 +1,34 @@
+module LlmLogs
+  module FormattingHelper
+    def format_duration_ms(ms)
+      return "—" if ms.nil?
+
+      "#{number_with_delimiter(sprintf('%.0f', ms))} ms"
+    end
+
+    def pretty_json(data)
+      JSON.pretty_generate(deep_parse_json(data))
+    rescue
+      data.to_s
+    end
+
+    private
+
+    def deep_parse_json(obj)
+      case obj
+      when Hash
+        obj.transform_values { |v| deep_parse_json(v) }
+      when Array
+        obj.map { |v| deep_parse_json(v) }
+      when String
+        begin
+          deep_parse_json(JSON.parse(obj))
+        rescue JSON::ParserError
+          obj
+        end
+      else
+        obj
+      end
+    end
+  end
+end

data/app/models/llm_logs/application_record.rb

@@ -0,0 +1,5 @@
+module LlmLogs
+  class ApplicationRecord < ActiveRecord::Base
+    self.abstract_class = true
+  end
+end

data/app/models/llm_logs/prompt.rb

@@ -0,0 +1,54 @@
+module LlmLogs
+  class Prompt < ApplicationRecord
+    has_many :versions, class_name: "LlmLogs::PromptVersion", dependent: :destroy
+
+    validates :slug, presence: true, uniqueness: true
+    validates :name, presence: true
+
+    def self.load(slug)
+      find_by!(slug: slug)
+    end
+
+    def current_version
+      versions.order(version_number: :desc).first
+    end
+
+    def version(number)
+      versions.find_by!(version_number: number)
+    end
+
+    def build(variables = {})
+      ver = current_version
+      raise "No versions exist for prompt '#{slug}'" unless ver
+
+      trace = LlmLogs::Tracer.current_trace
+      if trace && trace.prompt_version_id.nil?
+        trace.update_column(:prompt_version_id, ver.id)
+      end
+
+      ver.render(variables)
+    end
+
+    def update_content!(messages:, model: nil, model_params: {}, changelog: nil)
+      next_number = (versions.maximum(:version_number) || 0) + 1
+
+      versions.create!(
+        version_number: next_number,
+        messages: messages,
+        model: model,
+        model_params: model_params,
+        changelog: changelog
+      )
+    end
+
+    def rollback_to!(version_number)
+      source = version(version_number)
+      update_content!(
+        messages: source.messages,
+        model: source.model,
+        model_params: source.model_params,
+        changelog: "Rollback to version #{version_number}"
+      )
+    end
+  end
+end

data/app/models/llm_logs/prompt_version.rb

@@ -0,0 +1,31 @@
+require "mustache"
+
+module LlmLogs
+  class PromptVersion < ApplicationRecord
+    belongs_to :prompt
+    has_many :traces, class_name: "LlmLogs::Trace", dependent: :nullify
+
+    validates :version_number, presence: true, uniqueness: { scope: :prompt_id }
+    validates :messages, presence: true
+
+    def variables
+      messages.flat_map { |msg| msg["content"].to_s.scan(/\{\{[#^]?([^\/}]+)\}\}/) }.flatten.uniq.sort
+    end
+
+    def render(variables = {})
+      merged = (default_variables || {}).merge(variables.stringify_keys)
+
+      rendered_messages = messages.map do |msg|
+        {
+          role: msg["role"],
+          content: LlmLogs::PromptRenderer.render(msg["content"], merged)
+        }
+      end
+
+      params = { messages: rendered_messages }
+      params[:model] = model if model.present?
+      params.merge!(model_params.symbolize_keys) if model_params.present?
+      params
+    end
+  end
+end

data/app/models/llm_logs/span.rb

@@ -0,0 +1,38 @@
+module LlmLogs
+  class Span < ApplicationRecord
+    belongs_to :trace, counter_cache: true
+    belongs_to :parent_span, class_name: "LlmLogs::Span", optional: true
+    has_many :child_spans, class_name: "LlmLogs::Span", foreign_key: :parent_span_id, dependent: :nullify
+
+    validates :name, presence: true
+    validates :span_type, presence: true, inclusion: { in: %w[llm tool custom] }
+    validates :status, presence: true, inclusion: { in: %w[ok error] }
+    validates :started_at, presence: true
+
+    def finish
+      update!(
+        completed_at: Time.current,
+        duration_ms: (Time.current - started_at) * 1000
+      )
+
+      # Restore parent span as current
+      Thread.current[:llm_logs_span] = parent_span
+    end
+
+    def record_response(message)
+      self.output = { content: message.content.to_s }
+      self.input_tokens = message.input_tokens
+      self.output_tokens = message.output_tokens
+      self.cached_tokens = message.cached_tokens
+    end
+
+    def record_error(exception)
+      self.status = "error"
+      self.error_message = "#{exception.class}: #{exception.message}"
+    end
+
+    def set_attribute(key, value)
+      self.metadata = (metadata || {}).merge(key => value)
+    end
+  end
+end

data/app/models/llm_logs/trace.rb

@@ -0,0 +1,46 @@
+module LlmLogs
+  class Trace < ApplicationRecord
+    has_many :spans, dependent: :destroy
+    belongs_to :prompt_version, class_name: "LlmLogs::PromptVersion", optional: true
+
+    validates :name, presence: true
+    validates :status, presence: true, inclusion: { in: %w[running completed error] }
+    validates :started_at, presence: true
+
+    scope :recent, -> { order(started_at: :desc) }
+    scope :by_status, ->(status) { where(status: status) if status.present? }
+
+    def complete!
+      return if status == "completed"
+
+      rollup_stats!
+      update!(
+        status: "completed",
+        completed_at: Time.current,
+        duration_ms: (Time.current - started_at) * 1000
+      )
+    end
+
+    def fail!
+      rollup_stats!
+      update!(
+        status: "error",
+        completed_at: Time.current,
+        duration_ms: (Time.current - started_at) * 1000
+      )
+    end
+
+    def root_spans
+      spans.where(parent_span_id: nil).order(:started_at)
+    end
+
+    private
+
+    def rollup_stats!
+      self.total_input_tokens = spans.sum(:input_tokens)
+      self.total_output_tokens = spans.sum(:output_tokens)
+      self.total_cached_tokens = spans.sum(:cached_tokens)
+      self.total_cost = spans.sum(:cost)
+    end
+  end
+end

data/app/views/kaminari/tailwind/_first_page.html.erb

@@ -0,0 +1,5 @@
+<li>
+  <%= link_to url, class: "inline-flex items-center justify-center w-8 h-8 text-sm text-gray-700 bg-white border border-gray-300 rounded-md hover:bg-gray-50", remote: remote do %>
+    &laquo;
+  <% end %>
+</li>

data/app/views/kaminari/tailwind/_gap.html.erb

@@ -0,0 +1,3 @@
+<li>
+  <span class="inline-flex items-center justify-center w-8 h-8 text-sm text-gray-500">&hellip;</span>
+</li>

data/app/views/kaminari/tailwind/_last_page.html.erb

@@ -0,0 +1,5 @@
+<li>
+  <%= link_to url, class: "inline-flex items-center justify-center w-8 h-8 text-sm text-gray-700 bg-white border border-gray-300 rounded-md hover:bg-gray-50", remote: remote do %>
+    &raquo;
+  <% end %>
+</li>

data/app/views/kaminari/tailwind/_next_page.html.erb

@@ -0,0 +1,5 @@
+<li>
+  <%= link_to url, class: "inline-flex items-center justify-center w-8 h-8 text-sm text-gray-700 bg-white border border-gray-300 rounded-md hover:bg-gray-50", remote: remote do %>
+    &rsaquo;
+  <% end %>
+</li>

data/app/views/kaminari/tailwind/_page.html.erb

@@ -0,0 +1,9 @@
+<% if page.current? %>
+  <li>
+    <span class="inline-flex items-center justify-center w-8 h-8 text-sm font-medium text-white bg-indigo-600 rounded-md"><%= page %></span>
+  </li>
+<% else %>
+  <li>
+    <%= link_to page, url, class: "inline-flex items-center justify-center w-8 h-8 text-sm text-gray-700 bg-white border border-gray-300 rounded-md hover:bg-gray-50", remote: remote %>
+  </li>
+<% end %>

data/app/views/kaminari/tailwind/_paginator.html.erb

@@ -0,0 +1,17 @@
+<%= paginator.render do -%>
+  <nav class="flex items-center justify-center mt-6" aria-label="Pagination">
+    <ul class="flex items-center space-x-1">
+      <%= first_page_tag unless current_page.first? %>
+      <%= prev_page_tag unless current_page.first? %>
+      <% each_page do |page| -%>
+        <% if page.left_outer? || page.right_outer? || page.inside_window? -%>
+          <%= page_tag page %>
+        <% elsif !page.was_truncated? -%>
+          <%= gap_tag %>
+        <% end -%>
+      <% end -%>
+      <%= next_page_tag unless current_page.last? %>
+      <%= last_page_tag unless current_page.last? %>
+    </ul>
+  </nav>
+<% end -%>

data/app/views/kaminari/tailwind/_prev_page.html.erb

@@ -0,0 +1,5 @@
+<li>
+  <%= link_to url, class: "inline-flex items-center justify-center w-8 h-8 text-sm text-gray-700 bg-white border border-gray-300 rounded-md hover:bg-gray-50", remote: remote do %>
+    &lsaquo;
+  <% end %>
+</li>