llm_logs 0.1.0

data/app/views/llm_logs/traces/_span_node.html.erb

@@ -0,0 +1,66 @@
+<div class="<%= 'ml-6' if depth > 0 %> mb-2">
+  <details class="group">
+    <summary class="flex items-center justify-between p-3 rounded-lg bg-gray-50 hover:bg-gray-100 cursor-pointer">
+      <div class="flex items-center space-x-3">
+        <% type_colors = { "llm" => "bg-purple-100 text-purple-800", "tool" => "bg-amber-100 text-amber-800", "custom" => "bg-gray-100 text-gray-800" } %>
+        <span class="inline-flex items-center rounded px-1.5 py-0.5 text-xs font-medium <%= type_colors[span.span_type] %>">
+          <%= span.span_type %>
+        </span>
+        <span class="text-sm font-medium text-gray-900"><%= span.name %></span>
+        <% if span.model.present? %>
+          <span class="text-xs text-gray-500"><%= span.model %></span>
+        <% end %>
+      </div>
+      <div class="flex items-center space-x-4 text-xs text-gray-500">
+        <% if span.input_tokens || span.output_tokens %>
+          <span>
+            <%= span.input_tokens || 0 %> &rarr; <%= span.output_tokens || 0 %> tokens
+            <% if (span.cached_tokens || 0) > 0 %>
+              <span class="text-green-600">(<%= span.cached_tokens %> cached)</span>
+            <% end %>
+          </span>
+        <% end %>
+        <span><%= format_duration_ms(span.duration_ms) %></span>
+        <% if span.status == "error" %>
+          <span class="text-red-600 font-medium">ERROR</span>
+        <% end %>
+      </div>
+    </summary>
+
+    <div class="mt-2 ml-4 space-y-3">
+      <% if span.input.present? %>
+        <div>
+          <h4 class="text-xs font-medium text-gray-500 uppercase mb-1">Input</h4>
+          <pre class="text-xs bg-gray-50 rounded p-3 max-h-64 overflow-y-auto border"><%= pretty_json(span.input) %></pre>
+        </div>
+      <% end %>
+
+      <% if span.output.present? %>
+        <div>
+          <h4 class="text-xs font-medium text-gray-500 uppercase mb-1">Output</h4>
+          <pre class="text-xs bg-gray-50 rounded p-3 max-h-64 overflow-y-auto border"><%= pretty_json(span.output) %></pre>
+        </div>
+      <% end %>
+
+      <% if span.error_message.present? %>
+        <div>
+          <h4 class="text-xs font-medium text-red-600 uppercase mb-1">Error</h4>
+          <pre class="text-xs bg-red-50 rounded p-3 border border-red-200 text-red-800"><%= span.error_message %></pre>
+        </div>
+      <% end %>
+
+      <% if span.metadata.present? && span.metadata.any? %>
+        <div>
+          <h4 class="text-xs font-medium text-gray-500 uppercase mb-1">Metadata</h4>
+          <pre class="text-xs bg-gray-50 rounded p-3 border"><%= JSON.pretty_generate(span.metadata) %></pre>
+        </div>
+      <% end %>
+
+      <%= link_to "View details", trace_span_path(trace, span), class: "text-xs text-indigo-600 hover:text-indigo-900" %>
+    </div>
+  </details>
+
+  <% span.child_spans.order(:started_at).each do |child| %>
+    <%= render partial: "llm_logs/traces/span_node", locals: { span: child, depth: depth + 1, trace: trace } %>
+  <% end %>
+</div>

data/app/views/llm_logs/traces/index.html.erb

@@ -0,0 +1,78 @@
+<div class="flex items-center justify-between mb-6">
+  <h1 class="text-2xl font-bold text-gray-900">Traces</h1>
+  <% if @filter_version %>
+    <p class="text-sm text-gray-500">
+      Filtering by prompt: <span class="font-medium"><%= @filter_version.prompt.name %> v<%= @filter_version.version_number %></span>
+      &middot; <%= link_to "Clear filter", traces_path, class: "text-indigo-600 hover:text-indigo-900" %>
+    </p>
+  <% end %>
+
+  <div class="flex items-center space-x-3">
+    <%= form_tag traces_path, method: :get, class: "flex items-center space-x-2" do %>
+      <% if params[:prompt_version_id].present? %>
+        <input type="hidden" name="prompt_version_id" value="<%= params[:prompt_version_id] %>">
+      <% end %>
+      <select name="status" class="rounded-md border-gray-300 text-sm py-1.5 px-3 bg-white border shadow-sm">
+        <option value="">All statuses</option>
+        <% %w[running completed error].each do |status| %>
+          <option value="<%= status %>" <%= 'selected' if params[:status] == status %>><%= status %></option>
+        <% end %>
+      </select>
+
+      <button type="submit" class="bg-gray-900 text-white px-3 py-1.5 rounded-md text-sm hover:bg-gray-700">Filter</button>
+    <% end %>
+  </div>
+</div>
+
+<div class="bg-white shadow-sm ring-1 ring-gray-900/5 rounded-lg overflow-hidden">
+  <table class="min-w-full divide-y divide-gray-200">
+    <thead class="bg-gray-50">
+      <tr>
+        <th class="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">Name</th>
+        <th class="px-4 py-3 text-right text-xs font-medium text-gray-500 uppercase">Spans</th>
+        <th class="px-4 py-3 text-right text-xs font-medium text-gray-500 uppercase">Tokens</th>
+        <th class="px-4 py-3 text-right text-xs font-medium text-gray-500 uppercase">Cost</th>
+        <th class="px-4 py-3 text-right text-xs font-medium text-gray-500 uppercase">Duration</th>
+        <th class="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">Status</th>
+        <th class="px-4 py-3 text-left text-xs font-medium text-gray-500 uppercase">Started</th>
+      </tr>
+    </thead>
+    <tbody class="divide-y divide-gray-200">
+      <% @traces.each do |trace| %>
+        <tr class="hover:bg-gray-50">
+          <td class="px-4 py-3 text-sm">
+            <%= link_to trace.name, trace_path(trace), class: "text-indigo-600 hover:text-indigo-900 font-medium" %>
+          </td>
+          <td class="px-4 py-3 text-sm text-gray-500 text-right"><%= trace.spans_count %></td>
+          <td class="px-4 py-3 text-sm text-gray-500 text-right">
+            <%= number_with_delimiter(trace.total_input_tokens) %> &rarr; <%= number_with_delimiter(trace.total_output_tokens) %>
+            <% if trace.total_cached_tokens > 0 %>
+              <br><span class="text-xs text-green-600"><%= number_with_delimiter(trace.total_cached_tokens) %> cached</span>
+            <% end %>
+          </td>
+          <td class="px-4 py-3 text-sm text-gray-500 text-right">
+            <% if trace.total_cost > 0 %>$<%= sprintf('%.4f', trace.total_cost) %><% end %>
+          </td>
+          <td class="px-4 py-3 text-sm text-gray-500 text-right">
+            <%= format_duration_ms(trace.duration_ms) %>
+          </td>
+          <td class="px-4 py-3 text-sm">
+            <% status_colors = { "running" => "bg-blue-100 text-blue-800", "completed" => "bg-green-100 text-green-800", "error" => "bg-red-100 text-red-800" } %>
+            <span class="inline-flex items-center rounded-full px-2 py-0.5 text-xs font-medium <%= status_colors[trace.status] %>">
+              <%= trace.status %>
+            </span>
+          </td>
+          <td class="px-4 py-3 text-sm text-gray-500"><%= trace.started_at.strftime('%b %d %H:%M:%S') %></td>
+        </tr>
+      <% end %>
+
+      <% if @traces.empty? %>
+        <tr>
+          <td colspan="7" class="px-4 py-8 text-center text-sm text-gray-500">No traces found.</td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+</div>
+
+<%= paginate @traces, theme: "tailwind" %>

data/app/views/llm_logs/traces/show.html.erb

@@ -0,0 +1,79 @@
+<div class="mb-6">
+  <div class="flex items-center justify-between">
+    <div>
+      <div class="flex items-center space-x-3">
+        <h1 class="text-2xl font-bold text-gray-900"><%= @trace.name %></h1>
+        <% status_colors = { "running" => "bg-blue-100 text-blue-800", "completed" => "bg-green-100 text-green-800", "error" => "bg-red-100 text-red-800" } %>
+        <span class="inline-flex items-center rounded-full px-2.5 py-0.5 text-xs font-medium <%= status_colors[@trace.status] %>">
+          <%= @trace.status %>
+        </span>
+      </div>
+      <p class="text-sm text-gray-500 mt-1">
+        <%= @trace.started_at.strftime('%b %d, %Y %H:%M:%S') %>
+      </p>
+      <% if @trace.prompt_version.present? %>
+        <p class="text-sm text-gray-500 mt-1">
+          Prompt: <%= link_to "#{@trace.prompt_version.prompt.name} v#{@trace.prompt_version.version_number}",
+            prompt_version_path(@trace.prompt_version.prompt, @trace.prompt_version),
+            class: "text-indigo-600 hover:text-indigo-900" %>
+        </p>
+      <% end %>
+    </div>
+    <%= link_to "Back to traces", traces_path, class: "text-sm text-gray-600 hover:text-gray-900" %>
+  </div>
+</div>
+
+<div class="grid grid-cols-5 gap-4 mb-6">
+  <div class="bg-white rounded-lg p-4 shadow-sm ring-1 ring-gray-900/5">
+    <dt class="text-xs font-medium text-gray-500 uppercase">Spans</dt>
+    <dd class="text-2xl font-semibold text-gray-900 mt-1"><%= @trace.spans.count %></dd>
+  </div>
+  <div class="bg-white rounded-lg p-4 shadow-sm ring-1 ring-gray-900/5">
+    <dt class="text-xs font-medium text-gray-500 uppercase">Input Tokens</dt>
+    <dd class="text-2xl font-semibold text-gray-900 mt-1"><%= number_with_delimiter(@trace.total_input_tokens) %></dd>
+  </div>
+  <div class="bg-white rounded-lg p-4 shadow-sm ring-1 ring-gray-900/5">
+    <dt class="text-xs font-medium text-gray-500 uppercase">Output Tokens</dt>
+    <dd class="text-2xl font-semibold text-gray-900 mt-1"><%= number_with_delimiter(@trace.total_output_tokens) %></dd>
+  </div>
+  <div class="bg-white rounded-lg p-4 shadow-sm ring-1 ring-gray-900/5">
+    <dt class="text-xs font-medium text-gray-500 uppercase">Cached Tokens</dt>
+    <dd class="text-2xl font-semibold text-green-600 mt-1"><%= number_with_delimiter(@trace.total_cached_tokens) %></dd>
+    <% if @trace.total_input_tokens > 0 && @trace.total_cached_tokens > 0 %>
+      <dd class="text-xs text-green-600 mt-0.5"><%= (@trace.total_cached_tokens.to_f / @trace.total_input_tokens * 100).round %>% of input</dd>
+    <% end %>
+  </div>
+  <div class="bg-white rounded-lg p-4 shadow-sm ring-1 ring-gray-900/5">
+    <dt class="text-xs font-medium text-gray-500 uppercase">Duration</dt>
+    <dd class="text-2xl font-semibold text-gray-900 mt-1">
+      <%= format_duration_ms(@trace.duration_ms) %>
+    </dd>
+  </div>
+</div>
+
+<% if @trace.metadata.present? && @trace.metadata.any? %>
+  <div class="bg-white rounded-lg p-4 shadow-sm ring-1 ring-gray-900/5 mb-6">
+    <h3 class="text-sm font-medium text-gray-900 mb-2">Metadata</h3>
+    <dl class="grid grid-cols-2 gap-2 text-sm">
+      <% @trace.metadata.each do |key, value| %>
+        <dt class="text-gray-500"><%= key %></dt>
+        <dd class="text-gray-900"><%= value %></dd>
+      <% end %>
+    </dl>
+  </div>
+<% end %>
+
+<div class="bg-white rounded-lg shadow-sm ring-1 ring-gray-900/5">
+  <div class="px-4 py-3 border-b border-gray-200">
+    <h2 class="text-lg font-medium text-gray-900">Span Tree</h2>
+  </div>
+  <div class="p-4">
+    <% if @root_spans.any? %>
+      <% @root_spans.each do |span| %>
+        <%= render partial: "llm_logs/traces/span_node", locals: { span: span, depth: 0, trace: @trace } %>
+      <% end %>
+    <% else %>
+      <p class="text-sm text-gray-500">No spans recorded.</p>
+    <% end %>
+  </div>
+</div>

data/config/routes.rb

@@ -0,0 +1,18 @@
+LlmLogs::Engine.routes.draw do
+  root to: "traces#index"
+
+  resources :traces, only: [:index, :show] do
+    resources :spans, only: [:show]
+  end
+
+  resources :prompts do
+    resources :versions, only: [:index, :show, :destroy], controller: "prompt_versions" do
+      member do
+        post :restore
+      end
+      collection do
+        get :compare
+      end
+    end
+  end
+end

data/db/migrate/001_create_llm_logs_traces.rb

@@ -0,0 +1,22 @@
+class CreateLlmLogsTraces < ActiveRecord::Migration[8.0]
+  def change
+    create_table :llm_logs_traces do |t|
+      t.string :name, null: false
+      t.string :status, null: false, default: "running"
+      t.jsonb :metadata, default: {}
+      t.integer :total_input_tokens, default: 0
+      t.integer :total_output_tokens, default: 0
+      t.integer :total_cached_tokens, default: 0, null: false
+      t.decimal :total_cost, precision: 10, scale: 6, default: 0
+      t.float :duration_ms
+      t.integer :spans_count, default: 0, null: false
+      t.datetime :started_at, null: false
+      t.datetime :completed_at
+
+      t.timestamps
+    end
+
+    add_index :llm_logs_traces, :status
+    add_index :llm_logs_traces, :started_at
+  end
+end

data/db/migrate/002_create_llm_logs_spans.rb

@@ -0,0 +1,31 @@
+class CreateLlmLogsSpans < ActiveRecord::Migration[8.0]
+  def change
+    create_table :llm_logs_spans do |t|
+      t.references :trace, null: false, foreign_key: { to_table: :llm_logs_traces }
+      t.bigint :parent_span_id
+      t.string :name, null: false
+      t.string :span_type, null: false
+      t.string :model
+      t.string :provider
+      t.jsonb :input
+      t.jsonb :output
+      t.integer :input_tokens
+      t.integer :output_tokens
+      t.integer :cached_tokens
+      t.decimal :cost, precision: 10, scale: 6
+      t.float :duration_ms
+      t.string :status, null: false, default: "ok"
+      t.text :error_message
+      t.jsonb :metadata, default: {}
+      t.datetime :started_at, null: false
+      t.datetime :completed_at
+
+      t.timestamps
+    end
+
+    add_foreign_key :llm_logs_spans, :llm_logs_spans, column: :parent_span_id
+    add_index :llm_logs_spans, :parent_span_id
+    add_index :llm_logs_spans, :span_type
+    add_index :llm_logs_spans, :started_at
+  end
+end

data/db/migrate/003_create_llm_logs_prompts.rb

@@ -0,0 +1,13 @@
+class CreateLlmLogsPrompts < ActiveRecord::Migration[8.0]
+  def change
+    create_table :llm_logs_prompts do |t|
+      t.string :slug, null: false
+      t.string :name, null: false
+      t.text :description
+
+      t.timestamps
+    end
+
+    add_index :llm_logs_prompts, :slug, unique: true
+  end
+end

data/db/migrate/004_create_llm_logs_prompt_versions.rb

@@ -0,0 +1,18 @@
+class CreateLlmLogsPromptVersions < ActiveRecord::Migration[8.0]
+  def change
+    create_table :llm_logs_prompt_versions do |t|
+      t.references :prompt, null: false, foreign_key: { to_table: :llm_logs_prompts }
+      t.integer :version_number, null: false
+      t.jsonb :messages, null: false, default: []
+      t.string :model
+      t.jsonb :model_params, default: {}
+      t.jsonb :default_variables, default: {}
+      t.text :changelog
+
+      t.timestamps
+    end
+
+    add_index :llm_logs_prompt_versions, [:prompt_id, :version_number], unique: true,
+      name: "idx_llm_logs_prompt_versions_on_prompt_and_version"
+  end
+end

data/db/migrate/005_add_prompt_version_to_traces.rb

@@ -0,0 +1,7 @@
+class AddPromptVersionToTraces < ActiveRecord::Migration[8.0]
+  def change
+    add_reference :llm_logs_traces, :prompt_version,
+      foreign_key: { to_table: :llm_logs_prompt_versions, on_delete: :nullify },
+      null: true
+  end
+end

data/lib/generators/llm_logs/install_generator.rb

@@ -0,0 +1,17 @@
+module LlmLogs
+  class InstallGenerator < Rails::Generators::Base
+    source_root File.expand_path("templates", __dir__)
+
+    def copy_initializer
+      template "initializer.rb", "config/initializers/llm_logs.rb"
+    end
+
+    def mount_engine
+      route 'mount LlmLogs::Engine, at: "/llm_logs"'
+    end
+
+    def copy_migrations
+      rake "llm_logs:install:migrations"
+    end
+  end
+end

data/lib/generators/llm_logs/templates/initializer.rb

@@ -0,0 +1,10 @@
+LlmLogs.setup do |config|
+  # Set to false to disable auto-instrumentation of ruby_llm
+  # config.auto_instrument = false
+
+  # Set to false to completely disable logging
+  # config.enabled = false
+
+  # Number of days to keep trace data (for future cleanup job)
+  # config.retention_days = 30
+end

data/lib/llm_logs/engine.rb

@@ -0,0 +1,14 @@
+module LlmLogs
+  class Engine < ::Rails::Engine
+    isolate_namespace LlmLogs
+
+    initializer "llm_logs.auto_instrument" do
+      ActiveSupport.on_load(:active_record) do
+        if LlmLogs.auto_instrument && defined?(RubyLLM::Chat)
+          require "llm_logs/instrumentation/ruby_llm_chat"
+          RubyLLM::Chat.prepend(LlmLogs::Instrumentation::RubyLlmChat)
+        end
+      end
+    end
+  end
+end

data/lib/llm_logs/instrumentation/ruby_llm_chat.rb

@@ -0,0 +1,107 @@
+module LlmLogs
+  module Instrumentation
+    module RubyLlmChat
+      def complete(&block)
+        return super unless LlmLogs.enabled?
+
+        span = LlmLogs::Tracer.start_span(
+          name: "chat.complete",
+          span_type: "llm",
+          model: @model&.id,
+          provider: @model&.provider,
+          input: messages.map { |m| { role: m.role, content: llm_logs_serialize_content(m.content) } }
+        )
+
+        messages_before = messages.size
+
+        begin
+          result = super(&block)
+          span.record_response(result)
+          span.cost = llm_logs_compute_cost(result)
+          result
+        rescue => e
+          span.record_error(e)
+          llm_logs_capture_partial_tokens(span, messages_before)
+          raise
+        ensure
+          span.finish
+        end
+      end
+
+      private
+
+      def llm_logs_serialize_content(content)
+        content.is_a?(Hash) || content.is_a?(Array) ? content.to_json : content.to_s
+      end
+
+      def llm_logs_serialize_tool_result(result)
+        if result.is_a?(Hash)
+          result
+        elsif defined?(RubyLLM::Tool::Halt) && result.is_a?(RubyLLM::Tool::Halt)
+          { halted: result.message }
+        else
+          { result: result.to_s }
+        end
+      end
+
+      def llm_logs_capture_partial_tokens(span, messages_before)
+        assistant_msg = messages[messages_before..].find { |m| m.role == :assistant }
+        return unless assistant_msg&.input_tokens
+
+        span.input_tokens = assistant_msg.input_tokens
+        span.output_tokens = assistant_msg.output_tokens
+        span.cached_tokens = assistant_msg.cached_tokens
+        span.cost = llm_logs_compute_cost(assistant_msg)
+      rescue StandardError
+        nil
+      end
+
+      def llm_logs_compute_cost(message)
+        model_info = llm_logs_resolve_pricing_model
+        return unless model_info
+
+        input_price = model_info.input_price_per_million
+        output_price = model_info.output_price_per_million
+        return unless input_price && output_price
+
+        (message.input_tokens.to_f * input_price + message.output_tokens.to_f * output_price) / 1_000_000
+      end
+
+      def llm_logs_resolve_pricing_model
+        return @model if @model&.input_price_per_million
+
+        RubyLLM.models.find(@model.id) if @model&.id
+      rescue StandardError
+        nil
+      end
+
+      public
+
+      def execute_tool(tool_call)
+        return super unless LlmLogs.enabled?
+
+        span = LlmLogs::Tracer.start_span(
+          name: "tool.#{tool_call.name}",
+          span_type: "tool",
+          input: tool_call.arguments,
+          metadata: { tool_name: tool_call.name }
+        )
+
+        begin
+          result = super
+          span.output = llm_logs_serialize_tool_result(result)
+          span.set_attribute("tool.halted", true) if defined?(RubyLLM::Tool::Halt) && result.is_a?(RubyLLM::Tool::Halt)
+          # Convert hash/array results to JSON so RubyLLM stores clean JSON
+          # in message content, not Ruby hash syntax from Hash#to_s
+          result = result.to_json if result.is_a?(Hash) || result.is_a?(Array)
+          result
+        rescue => e
+          span.record_error(e)
+          raise
+        ensure
+          span.finish
+        end
+      end
+    end
+  end
+end

data/lib/llm_logs/prompt_renderer.rb

@@ -0,0 +1,17 @@
+require "mustache"
+
+module LlmLogs
+  class PromptRenderer < Mustache
+    # Disable HTML escaping — LLM prompts are plain text
+    def escapeHTML(str)
+      str
+    end
+
+    def self.render(template_string, variables = {})
+      renderer = new
+      renderer.template = template_string
+      variables.each { |key, value| renderer[key] = value }
+      renderer.render
+    end
+  end
+end

data/lib/llm_logs/tracer.rb

@@ -0,0 +1,67 @@
+module LlmLogs
+  module Tracer
+    def self.current_trace
+      Thread.current[:llm_logs_trace]
+    end
+
+    def self.current_span
+      Thread.current[:llm_logs_span]
+    end
+
+    def self.start_trace(name, metadata: {})
+      trace = LlmLogs::Trace.create!(
+        name: name,
+        status: "running",
+        metadata: metadata,
+        started_at: Time.current
+      )
+
+      previous_trace = Thread.current[:llm_logs_trace]
+      previous_span = Thread.current[:llm_logs_span]
+      Thread.current[:llm_logs_trace] = trace
+      Thread.current[:llm_logs_span] = nil
+
+      begin
+        yield trace
+      rescue => e
+        trace.fail!
+        raise
+      ensure
+        trace.complete! if trace.status == "running"
+        Thread.current[:llm_logs_trace] = previous_trace
+        Thread.current[:llm_logs_span] = previous_span
+      end
+    end
+
+    def self.start_span(name:, span_type:, model: nil, provider: nil, input: nil, metadata: {})
+      trace = current_trace || auto_create_trace(name)
+      parent = current_span
+
+      span = LlmLogs::Span.create!(
+        trace: trace,
+        parent_span: parent,
+        name: name,
+        span_type: span_type,
+        model: model,
+        provider: provider,
+        input: input,
+        metadata: metadata,
+        status: "ok",
+        started_at: Time.current
+      )
+
+      Thread.current[:llm_logs_span] = span
+      span
+    end
+
+    def self.auto_create_trace(span_name)
+      trace = LlmLogs::Trace.create!(
+        name: "auto:#{span_name}",
+        status: "running",
+        started_at: Time.current
+      )
+      Thread.current[:llm_logs_trace] = trace
+      trace
+    end
+  end
+end

data/lib/llm_logs/version.rb

@@ -0,0 +1,3 @@
+module LlmLogs
+  VERSION = "0.1.0"
+end

data/lib/llm_logs.rb

@@ -0,0 +1,23 @@
+require "kaminari"
+require "llm_logs/version"
+require "llm_logs/engine"
+require "llm_logs/tracer"
+require "llm_logs/prompt_renderer"
+
+module LlmLogs
+  mattr_accessor :enabled, default: true
+  mattr_accessor :auto_instrument, default: true
+  mattr_accessor :retention_days, default: 30
+
+  def self.enabled?
+    enabled
+  end
+
+  def self.setup
+    yield self
+  end
+
+  def self.trace(name, **options, &block)
+    LlmLogs::Tracer.start_trace(name, **options, &block)
+  end
+end