rails_error_dashboard 0.6.3 → 0.7.0

data/lib/rails_error_dashboard/subscribers/llm_call_subscriber.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Subscribers
+    # ActiveSupport::Notifications subscriber for manually-instrumented LLM
+    # calls. The Tier 3 path — for hosts that don't run OpenTelemetry AND
+    # don't use Faraday-based LLM SDKs (e.g., direct Net::HTTP, gRPC clients,
+    # custom adapters, or hosts that want to layer in extra LLM activity that
+    # the automatic paths can't see, like local inference servers).
+    #
+    # Usage in the host app:
+    #
+    #   ActiveSupport::Notifications.instrument("red.llm_call",
+    #     provider: "ollama",
+    #     model: "llama3:8b",
+    #     input_tokens: 1200,
+    #     output_tokens: 350
+    #   ) do
+    #     # ... call your LLM ...
+    #   end
+    #
+    #   # Tool execution
+    #   ActiveSupport::Notifications.instrument("red.llm_tool_call",
+    #     tool_name: "search_database",
+    #     tool_arguments: { query: "..." },
+    #     tool_result: "[...]"
+    #   ) do
+    #     # ... execute tool ...
+    #   end
+    #
+    # Payload contract = LlmCallEvent constructor kwargs:
+    #   :provider, :model, :status, :input_tokens, :output_tokens,
+    #   :duration_ms, :error_class, :error_message, :tool_name,
+    #   :tool_arguments, :tool_result, :cost_usd_estimate
+    #
+    # Duration: defaults to `event.duration` (host wraps `.instrument` around
+    # the work); payload `:duration_ms` overrides if explicitly supplied.
+    # Cost: auto-estimated from provider/model/tokens unless payload supplies
+    # `:cost_usd_estimate`.
+    #
+    # SAFETY RULES (HOST_APP_SAFETY.md):
+    # - Every callback wrapped in rescue => e; nil
+    # - Never raise from subscriber callbacks
+    # - Skip if buffer is nil (not in a request context)
+    # - Re-read config on every event (host may toggle at runtime)
+    class LlmCallSubscriber
+      CHAT_EVENT = "red.llm_call"
+      TOOL_EVENT = "red.llm_tool_call"
+      EVENTS = [ CHAT_EVENT, TOOL_EVENT ].freeze
+
+      @subscriptions = []
+
+      class << self
+        attr_reader :subscriptions
+
+        # Idempotent — re-subscribing first tears down previous subscriptions
+        # so Spring reloads / repeated engine boots don't pile up duplicates.
+        def subscribe!
+          unsubscribe!
+          @subscriptions = EVENTS.map { |name| subscribe_event(name) }
+        end
+
+        def unsubscribe!
+          (@subscriptions || []).each do |sub|
+            ActiveSupport::Notifications.unsubscribe(sub) if sub
+          rescue StandardError
+            nil
+          end
+          @subscriptions = []
+        end
+
+        private
+
+        def subscribe_event(event_name)
+          ActiveSupport::Notifications.subscribe(event_name) do |*args|
+            event = ActiveSupport::Notifications::Event.new(*args)
+            handle_event(event, event_name)
+          rescue StandardError => e
+            RailsErrorDashboard::Logger.debug("[RailsErrorDashboard] LlmCallSubscriber callback failed: #{e.message}")
+            nil
+          end
+        end
+
+        def handle_event(event, event_name)
+          return unless RailsErrorDashboard.configuration.enable_llm_observability
+          return unless RailsErrorDashboard.configuration.enable_breadcrumbs
+          return unless Services::BreadcrumbCollector.current_buffer
+
+          payload = event.payload || {}
+          # Symbolize-ish access — hosts may pass either symbol or string keys
+          payload = payload.transform_keys(&:to_sym) if payload.respond_to?(:transform_keys)
+
+          tool_name = payload[:tool_name] || (event_name == TOOL_EVENT ? "unknown" : nil)
+          provider  = payload[:provider]
+          model     = payload[:model]
+
+          duration_ms = payload[:duration_ms]
+          duration_ms ||= event.duration if event.respond_to?(:duration)
+
+          status = normalize_status(payload[:status], payload[:error_class])
+
+          cost = payload[:cost_usd_estimate]
+          if cost.nil? && tool_name.nil? && status == :success && model
+            cost = Services::LlmCostEstimator.estimate(
+              provider: provider,
+              model: model,
+              input_tokens: payload[:input_tokens],
+              output_tokens: payload[:output_tokens]
+            )
+          end
+
+          llm_event = ValueObjects::LlmCallEvent.new(
+            provider: provider || "unknown",
+            model: model || "unknown",
+            status: status,
+            input_tokens: payload[:input_tokens],
+            output_tokens: payload[:output_tokens],
+            duration_ms: duration_ms,
+            error_class: payload[:error_class],
+            error_message: payload[:error_message],
+            tool_name: tool_name,
+            tool_arguments: payload[:tool_arguments],
+            tool_result: payload[:tool_result],
+            cost_usd_estimate: cost
+          )
+
+          category = llm_event.tool_call? ? "llm_tool" : "llm"
+
+          Services::BreadcrumbCollector.add(
+            category,
+            llm_event.to_breadcrumb_message,
+            duration_ms: llm_event.duration_ms,
+            metadata: llm_event.to_breadcrumb_metadata
+          )
+        end
+
+        # Status precedence: explicit payload status (if valid) → :error when
+        # error_class present → :success.
+        def normalize_status(payload_status, error_class)
+          if payload_status
+            sym = payload_status.to_sym rescue nil
+            return sym if ValueObjects::LlmCallEvent::STATUSES.include?(sym)
+          end
+          return :error if error_class
+          :success
+        end
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/value_objects/llm_call_event.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module ValueObjects
+    # Immutable value object representing a single LLM call observed in the
+    # host application. One canonical shape that all capture paths (OTel
+    # SpanProcessor, Faraday middleware, future native subscribers) normalize
+    # to before handing off to BreadcrumbCollector.
+    #
+    # Fields are read-only after initialization. Unknown / unavailable fields
+    # are nil — never raise, never block, never allocate large strings.
+    class LlmCallEvent
+      STATUSES = [ :success, :error, :timeout ].freeze
+      MAX_TOOL_ARG_LENGTH = 500
+      MAX_TOOL_RESULT_LENGTH = 500
+      MAX_ERROR_MESSAGE_LENGTH = 200
+
+      attr_reader :provider, :model, :input_tokens, :output_tokens,
+                  :duration_ms, :status, :error_class, :error_message,
+                  :tool_name, :tool_arguments_truncated, :tool_result_truncated,
+                  :cost_usd_estimate
+
+      def initialize(provider:, model:, status:,
+                     input_tokens: nil, output_tokens: nil, duration_ms: nil,
+                     error_class: nil, error_message: nil,
+                     tool_name: nil, tool_arguments: nil, tool_result: nil,
+                     cost_usd_estimate: nil)
+        @provider = provider.to_s
+        @model = model.to_s
+        @status = STATUSES.include?(status) ? status : :success
+        @input_tokens = input_tokens
+        @output_tokens = output_tokens
+        @duration_ms = duration_ms
+        @error_class = error_class
+        @error_message = truncate(error_message, MAX_ERROR_MESSAGE_LENGTH)
+        @tool_name = tool_name
+        @tool_arguments_truncated = truncate(tool_arguments, MAX_TOOL_ARG_LENGTH)
+        @tool_result_truncated = truncate(tool_result, MAX_TOOL_RESULT_LENGTH)
+        @cost_usd_estimate = cost_usd_estimate
+        freeze
+      end
+
+      def tool_call?
+        !@tool_name.nil?
+      end
+
+      # Hash shape passed to BreadcrumbCollector.add(..., metadata:).
+      # Only includes non-nil keys — keeps the breadcrumb JSON compact.
+      def to_breadcrumb_metadata
+        {
+          provider: @provider,
+          model: @model,
+          status: @status.to_s,
+          input_tokens: @input_tokens,
+          output_tokens: @output_tokens,
+          duration_ms: @duration_ms,
+          error_class: @error_class,
+          error_message: @error_message,
+          tool_name: @tool_name,
+          tool_arguments: @tool_arguments_truncated,
+          tool_result: @tool_result_truncated,
+          cost_usd: @cost_usd_estimate
+        }.compact
+      end
+
+      # Short human-readable message for the breadcrumb (rendered in UI).
+      def to_breadcrumb_message
+        if tool_call?
+          "tool: #{@tool_name}"
+        else
+          parts = [ @provider, @model ]
+          if @input_tokens && @output_tokens
+            parts << "in:#{@input_tokens}/out:#{@output_tokens}"
+          end
+          parts << @status.to_s if @status != :success
+          parts.compact.join(" · ")
+        end
+      end
+
+      private
+
+      def truncate(value, limit)
+        return nil if value.nil?
+        str = value.to_s
+        return str if str.length <= limit
+        "#{str[0, limit]}…"
+      rescue
+        nil
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/version.rb

@@ -1,3 +1,3 @@
 module RailsErrorDashboard
-  VERSION = "0.6.3"
+  VERSION = "0.7.0"
 end

data/lib/rails_error_dashboard.rb

@@ -19,7 +19,12 @@ begin; require "turbo-rails"; rescue LoadError; end
 
 # Core library files
 require "rails_error_dashboard/value_objects/error_context"
+require "rails_error_dashboard/value_objects/llm_call_event"
+require "rails_error_dashboard/integrations/o_tel"
+require "rails_error_dashboard/integrations/llm_span_processor"
+require "rails_error_dashboard/integrations/llm_middleware"
 require "rails_error_dashboard/helpers/user_model_detector"
+require "rails_error_dashboard/services/llm_cost_estimator"
 require "rails_error_dashboard/services/platform_detector"
 require "rails_error_dashboard/services/backtrace_parser"
 require "rails_error_dashboard/services/similarity_calculator"
@@ -57,12 +62,14 @@ require "rails_error_dashboard/services/n_plus_one_detector"
 require "rails_error_dashboard/services/curl_generator"
 require "rails_error_dashboard/services/rspec_generator"
 require "rails_error_dashboard/services/markdown_error_formatter"
+require "rails_error_dashboard/services/llm_client"
 require "rails_error_dashboard/services/issue_tracker_client"
 require "rails_error_dashboard/services/github_issue_client"
 require "rails_error_dashboard/services/gitlab_issue_client"
 require "rails_error_dashboard/services/codeberg_issue_client"
 require "rails_error_dashboard/services/database_health_inspector"
 require "rails_error_dashboard/services/cache_analyzer"
+require "rails_error_dashboard/services/llm_summary"
 require "rails_error_dashboard/services/variable_serializer"
 require "rails_error_dashboard/services/local_variable_capturer"
 require "rails_error_dashboard/services/swallowed_exception_tracker"
@@ -75,6 +82,7 @@ require "rails_error_dashboard/subscribers/breadcrumb_subscriber"
 require "rails_error_dashboard/subscribers/rack_attack_subscriber"
 require "rails_error_dashboard/subscribers/action_cable_subscriber"
 require "rails_error_dashboard/subscribers/active_storage_subscriber"
+require "rails_error_dashboard/subscribers/llm_call_subscriber"
 require "rails_error_dashboard/subscribers/issue_tracker_subscriber"
 require "rails_error_dashboard/queries/co_occurring_errors"
 require "rails_error_dashboard/queries/action_cable_summary"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rails_error_dashboard
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.7.0
 platform: ruby
 authors:
 - Anjan Jagirdar
@@ -281,6 +281,7 @@ files:
 - app/views/rails_error_dashboard/digest_mailer/digest_summary.text.erb
 - app/views/rails_error_dashboard/error_notification_mailer/error_alert.html.erb
 - app/views/rails_error_dashboard/error_notification_mailer/error_alert.text.erb
+- app/views/rails_error_dashboard/errors/_ai_help_panel.html.erb
 - app/views/rails_error_dashboard/errors/_breadcrumbs_group.html.erb
 - app/views/rails_error_dashboard/errors/_co_occurring_errors.html.erb
 - app/views/rails_error_dashboard/errors/_discussion.html.erb
@@ -289,6 +290,7 @@ files:
 - app/views/rails_error_dashboard/errors/_error_row.html.erb
 - app/views/rails_error_dashboard/errors/_instance_variables.html.erb
 - app/views/rails_error_dashboard/errors/_issue_section.html.erb
+- app/views/rails_error_dashboard/errors/_llm_summary.html.erb
 - app/views/rails_error_dashboard/errors/_local_variables.html.erb
 - app/views/rails_error_dashboard/errors/_modals.html.erb
 - app/views/rails_error_dashboard/errors/_pattern_insights.html.erb
@@ -392,6 +394,9 @@ files:
 - lib/rails_error_dashboard/engine.rb
 - lib/rails_error_dashboard/error_reporter.rb
 - lib/rails_error_dashboard/helpers/user_model_detector.rb
+- lib/rails_error_dashboard/integrations/llm_middleware.rb
+- lib/rails_error_dashboard/integrations/llm_span_processor.rb
+- lib/rails_error_dashboard/integrations/o_tel.rb
 - lib/rails_error_dashboard/logger.rb
 - lib/rails_error_dashboard/manual_error_reporter.rb
 - lib/rails_error_dashboard/middleware/error_catcher.rb
@@ -455,6 +460,9 @@ files:
 - lib/rails_error_dashboard/services/gitlab_issue_client.rb
 - lib/rails_error_dashboard/services/issue_body_formatter.rb
 - lib/rails_error_dashboard/services/issue_tracker_client.rb
+- lib/rails_error_dashboard/services/llm_client.rb
+- lib/rails_error_dashboard/services/llm_cost_estimator.rb
+- lib/rails_error_dashboard/services/llm_summary.rb
 - lib/rails_error_dashboard/services/local_variable_capturer.rb
 - lib/rails_error_dashboard/services/markdown_error_formatter.rb
 - lib/rails_error_dashboard/services/n_plus_one_detector.rb
@@ -480,9 +488,11 @@ files:
 - lib/rails_error_dashboard/subscribers/active_storage_subscriber.rb
 - lib/rails_error_dashboard/subscribers/breadcrumb_subscriber.rb
 - lib/rails_error_dashboard/subscribers/issue_tracker_subscriber.rb
+- lib/rails_error_dashboard/subscribers/llm_call_subscriber.rb
 - lib/rails_error_dashboard/subscribers/rack_attack_subscriber.rb
 - lib/rails_error_dashboard/test_error.rb
 - lib/rails_error_dashboard/value_objects/error_context.rb
+- lib/rails_error_dashboard/value_objects/llm_call_event.rb
 - lib/rails_error_dashboard/version.rb
 - lib/tasks/error_dashboard.rake
 - lib/tasks/rails_error_dashboard_tasks.rake
@@ -498,7 +508,7 @@ metadata:
   funding_uri: https://github.com/sponsors/AnjanJ
 post_install_message: |
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
-    RED (Rails Error Dashboard) v0.6.3
+    RED (Rails Error Dashboard) v0.7.0
   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 
   First install:
@@ -534,7 +544,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Self-hosted error tracking and exception monitoring for Rails. Free, forever.
 test_files: []