rails_error_dashboard 0.6.4 → 0.7.0

data/lib/rails_error_dashboard/services/llm_client.rb

@@ -0,0 +1,368 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "uri"
+
+module RailsErrorDashboard
+  module Services
+    class LlmClient
+      class ConfigurationError < StandardError; end
+      class RequestError < StandardError; end
+
+      OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses"
+      OPENAI_CHAT_COMPLETIONS_URL = "https://api.openai.com/v1/chat/completions"
+      ANTHROPIC_MESSAGES_URL = "https://api.anthropic.com/v1/messages"
+      ANTHROPIC_VERSION = "2023-06-01"
+
+      attr_reader :config
+
+      def self.call(error:, question:, context:)
+        new.call(error: error, question: question, context: context)
+      end
+
+      def self.stream(error:, question:, context:, &block)
+        new.stream(error: error, question: question, context: context, &block)
+      end
+
+      def initialize(config: RailsErrorDashboard.configuration)
+        @config = config
+      end
+
+      def call(error:, question:, context:)
+        raise ConfigurationError, "AI Help is not configured" unless config.llm_configured?
+
+        case config.effective_llm_provider
+        when :openai
+          openai_response(error: error, question: question, context: context)
+        when :anthropic
+          anthropic_response(error: error, question: question, context: context)
+        else
+          raise ConfigurationError, "Unsupported LLM provider: #{config.llm_provider.inspect}"
+        end
+      end
+
+      def stream(error:, question:, context:, &block)
+        raise ConfigurationError, "AI Help is not configured" unless config.llm_configured?
+        raise ArgumentError, "block is required for streaming" unless block
+
+        case config.effective_llm_provider
+        when :openai
+          openai_stream(error: error, question: question, context: context, &block)
+        when :anthropic
+          anthropic_stream(error: error, question: question, context: context, &block)
+        else
+          raise ConfigurationError, "Unsupported LLM provider: #{config.llm_provider.inspect}"
+        end
+      end
+
+      private
+
+      def openai_response(error:, question:, context:)
+        model = config.effective_llm_model
+        mode = openai_endpoint_for(model)
+
+        if mode == :chat_completions
+          response = post_json(
+            OPENAI_CHAT_COMPLETIONS_URL,
+            openai_headers,
+            openai_chat_payload(error: error, question: question, context: context, model: model)
+          )
+          answer = response.dig("choices", 0, "message", "content")
+        else
+          response = post_json(
+            OPENAI_RESPONSES_URL,
+            openai_headers,
+            openai_responses_payload(error: error, question: question, context: context, model: model)
+          )
+          answer = extract_openai_response_text(response)
+        end
+
+        provider_result(answer, :openai, model)
+      end
+
+      def anthropic_response(error:, question:, context:)
+        model = config.effective_llm_model
+        response = post_json(
+          ANTHROPIC_MESSAGES_URL,
+          anthropic_headers,
+          anthropic_payload(error: error, question: question, context: context, model: model)
+        )
+        answer = Array(response["content"]).filter_map { |part| part["text"] if part["type"] == "text" }.join("\n\n")
+
+        provider_result(answer, :anthropic, model)
+      end
+
+      def openai_stream(error:, question:, context:)
+        model = config.effective_llm_model
+        mode = openai_endpoint_for(model)
+
+        if mode == :chat_completions
+          payload = openai_chat_payload(error: error, question: question, context: context, model: model).merge(stream: true)
+          post_stream_json(OPENAI_CHAT_COMPLETIONS_URL, openai_headers, payload) do |event, data|
+            handle_openai_stream_event(event, data) { |text| yield text }
+          end
+        else
+          payload = openai_responses_payload(error: error, question: question, context: context, model: model).merge(stream: true)
+          post_stream_json(OPENAI_RESPONSES_URL, openai_headers, payload) do |event, data|
+            handle_openai_stream_event(event, data) { |text| yield text }
+          end
+        end
+
+        { provider: "openai", model: model }
+      end
+
+      def anthropic_stream(error:, question:, context:)
+        model = config.effective_llm_model
+        payload = anthropic_payload(error: error, question: question, context: context, model: model).merge(stream: true)
+
+        post_stream_json(ANTHROPIC_MESSAGES_URL, anthropic_headers, payload) do |event, data|
+          handle_anthropic_stream_event(event, data) { |text| yield text }
+        end
+
+        { provider: "anthropic", model: model }
+      end
+
+      def openai_endpoint_for(model)
+        configured = config.llm_openai_endpoint&.to_sym || :auto
+        return configured unless configured == :auto
+
+        :responses
+      end
+
+      def openai_headers
+        {
+          "Authorization" => "Bearer #{config.effective_llm_api_key}",
+          "Content-Type" => "application/json"
+        }
+      end
+
+      def anthropic_headers
+        {
+          "x-api-key" => config.effective_llm_api_key,
+          "anthropic-version" => ANTHROPIC_VERSION,
+          "Content-Type" => "application/json"
+        }
+      end
+
+      def openai_responses_payload(error:, question:, context:, model:)
+        payload = {
+          model: model,
+          instructions: system_prompt,
+          input: user_prompt(error: error, question: question, context: context),
+          max_output_tokens: config.llm_max_output_tokens.to_i
+        }
+
+        payload
+      end
+
+      def openai_chat_payload(error:, question:, context:, model:)
+        {
+          model: model,
+          messages: [
+            { role: "system", content: system_prompt },
+            { role: "user", content: user_prompt(error: error, question: question, context: context) }
+          ],
+          max_tokens: config.llm_max_output_tokens.to_i
+        }
+      end
+
+      def anthropic_payload(error:, question:, context:, model:)
+        {
+          model: model,
+          max_tokens: config.llm_max_output_tokens.to_i,
+          system: system_prompt,
+          messages: [
+            { role: "user", content: user_prompt(error: error, question: question, context: context) }
+          ]
+        }
+      end
+
+      def system_prompt
+        prompt = <<~PROMPT.strip
+          You are helping debug a Rails exception from Rails Error Dashboard.
+          Answer only from the provided error context unless you clearly label an inference.
+          Focus on likely root cause, useful next checks, and concrete Rails code or data fixes.
+          Do not ask for secrets, credentials, or unrelated source code.
+        PROMPT
+
+        [ prompt, config.llm_system_prompt.presence ].compact.join("\n\n")
+      end
+
+      def user_prompt(error:, question:, context:)
+        <<~PROMPT
+          Error ID: #{error.id}
+          Error type: #{error.error_type}
+          Severity: #{error.severity}
+          Occurrences: #{error.occurrence_count}
+
+          User question:
+          #{question}
+
+          Error context:
+          #{context}
+        PROMPT
+      end
+
+      def post_json(url, headers, payload)
+        uri = URI.parse(url)
+        request = Net::HTTP::Post.new(uri.request_uri, headers)
+        request.body = JSON.generate(payload)
+
+        response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
+          open_timeout: config.llm_timeout_seconds.to_i,
+          read_timeout: config.llm_timeout_seconds.to_i) do |http|
+          http.request(request)
+        end
+
+        parsed = parse_json_response(response)
+        return parsed if response.is_a?(Net::HTTPSuccess)
+
+        message = provider_error_message(parsed, response.message)
+        raise RequestError, "LLM request failed (#{response.code}): #{message}"
+      rescue JSON::ParserError
+        raise RequestError, "LLM provider returned invalid JSON"
+      rescue Net::OpenTimeout, Net::ReadTimeout
+        raise RequestError, "LLM request timed out"
+      rescue SocketError, SystemCallError => e
+        raise RequestError, "LLM request failed: #{e.message}"
+      end
+
+      def parse_json_response(response)
+        body = response.body.to_s
+        body.present? ? JSON.parse(body) : {}
+      end
+
+      def extract_openai_response_text(response)
+        return response["output_text"] if response["output_text"].present?
+
+        Array(response["output"]).flat_map do |item|
+          Array(item["content"]).filter_map { |part| part["text"] if part["type"] == "output_text" || part["type"] == "text" }
+        end.join("\n\n")
+      end
+
+      def provider_result(answer, provider, model)
+        raise RequestError, "LLM provider returned an empty answer" if answer.blank?
+
+        { answer: answer, provider: provider.to_s, model: model }
+      end
+
+      def post_stream_json(url, headers, payload)
+        uri = URI.parse(url)
+        stream_headers = headers.merge(
+          "Accept" => "text/event-stream",
+          "Accept-Encoding" => "identity",
+          "Cache-Control" => "no-cache"
+        )
+        request = Net::HTTP::Post.new(uri.request_uri, stream_headers)
+        request.body = JSON.generate(payload)
+
+        Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https",
+          open_timeout: config.llm_timeout_seconds.to_i,
+          read_timeout: config.llm_timeout_seconds.to_i) do |http|
+          http.request(request) do |response|
+            unless response.is_a?(Net::HTTPSuccess)
+              body = +""
+              response.read_body { |chunk| body << chunk }
+              parsed = JSON.parse(body) rescue {}
+              message = provider_error_message(parsed, response.message)
+              raise RequestError, "LLM request failed (#{response.code}): #{message}"
+            end
+
+            each_sse_event(response) do |event, data|
+              next if data == "[DONE]"
+
+              parsed = JSON.parse(data)
+              yield event, parsed
+            rescue JSON::ParserError
+              raise RequestError, "LLM provider returned invalid streaming JSON"
+            end
+          end
+        end
+      rescue Net::OpenTimeout, Net::ReadTimeout
+        raise RequestError, "LLM request timed out"
+      rescue SocketError, SystemCallError => e
+        raise RequestError, "LLM request failed: #{e.message}"
+      end
+
+      def each_sse_event(response)
+        buffer = +""
+
+        response.read_body do |chunk|
+          buffer << chunk.to_s
+          buffer.gsub!("\r\n", "\n")
+          while (separator = buffer.index("\n\n"))
+            raw_event = buffer.slice!(0, separator + 2)
+            event, data = parse_sse_event(raw_event)
+            yield event, data if data.present?
+          end
+        end
+
+        event, data = parse_sse_event(buffer)
+        yield event, data if data.present?
+      end
+
+      def parse_sse_event(raw_event)
+        event = "message"
+        data_lines = []
+
+        raw_event.to_s.each_line do |line|
+          line = line.chomp
+          next if line.blank? || line.start_with?(":")
+
+          field, value = line.split(":", 2)
+          value = value.to_s.sub(/\A /, "")
+
+          case field
+          when "event"
+            event = value
+          when "data"
+            data_lines << value
+          end
+        end
+
+        [ event, data_lines.join("\n") ]
+      end
+
+      def handle_openai_stream_event(event, data)
+        if event == "error" || data["type"] == "error"
+          message = provider_error_message(data, "OpenAI stream failed")
+          raise RequestError, message
+        end
+
+        text = if event == "response.output_text.delta" || data["type"] == "response.output_text.delta"
+          data["delta"]
+        else
+          data.dig("choices", 0, "delta", "content")
+        end
+
+        yield text if text.present?
+      end
+
+      def handle_anthropic_stream_event(event, data)
+        if event == "error" || data["type"] == "error"
+          message = provider_error_message(data, "Anthropic stream failed")
+          raise RequestError, message
+        end
+
+        delta = data["delta"] || {}
+        text = delta["text"] if data["type"] == "content_block_delta" && delta["type"] == "text_delta"
+        yield text if text.present?
+      end
+
+      def provider_error_message(parsed, fallback)
+        return fallback unless parsed.is_a?(Hash)
+
+        error = parsed["error"]
+        case error
+        when Hash
+          error["message"].presence || parsed["message"].presence || fallback
+        when String
+          error.presence || parsed["message"].presence || fallback
+        else
+          parsed["message"].presence || fallback
+        end
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/services/llm_cost_estimator.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Services
+    # Estimates USD cost for an LLM call given provider, model, and token counts.
+    #
+    # Prices are stored as USD per 1,000,000 tokens (the canonical unit used
+    # by OpenAI, Anthropic, and Google in their pricing pages).
+    #
+    # IMPORTANT: This is a best-effort estimate using a hardcoded snapshot.
+    # Prices change frequently. Users SHOULD configure overrides via
+    # `config.llm_pricing_overrides` for production accuracy.
+    #
+    # Returns nil for unknown models (no override AND not in the built-in table).
+    # Never raises — wrapped in rescue.
+    class LlmCostEstimator
+      # Prices last refreshed: 2026-05 (approximate, see provider pricing pages).
+      # Format: { "model-name" => { input: usd_per_1m, output: usd_per_1m } }
+      PRICES = {
+        # Anthropic
+        "claude-opus-4-7" => { input: 15.0, output: 75.0 },
+        "claude-sonnet-4-6" => { input: 3.0, output: 15.0 },
+        "claude-haiku-4-5" => { input: 0.80, output: 4.0 },
+
+        # OpenAI
+        "gpt-4o" => { input: 2.50, output: 10.0 },
+        "gpt-4o-mini" => { input: 0.15, output: 0.60 },
+        "gpt-4-turbo" => { input: 10.0, output: 30.0 },
+        "o1" => { input: 15.0, output: 60.0 },
+        "o1-mini" => { input: 3.0, output: 12.0 },
+
+        # Google
+        "gemini-2.5-pro" => { input: 1.25, output: 5.0 },
+        "gemini-2.5-flash" => { input: 0.075, output: 0.30 }
+      }.freeze
+
+      # @param provider [String, Symbol] currently informational only (not used in lookup)
+      # @param model [String] model identifier — matched case-insensitively against PRICES
+      # @param input_tokens [Integer, nil]
+      # @param output_tokens [Integer, nil]
+      # @return [Float, nil] estimated USD cost, or nil if model unknown / tokens missing
+      def self.estimate(provider:, model:, input_tokens:, output_tokens:)
+        return nil if model.nil? || model.to_s.empty?
+        return nil if input_tokens.nil? && output_tokens.nil?
+
+        rate = lookup_rate(model.to_s)
+        return nil unless rate
+
+        in_tokens = input_tokens.to_i
+        out_tokens = output_tokens.to_i
+
+        cost = (in_tokens * rate[:input] + out_tokens * rate[:output]) / 1_000_000.0
+        cost.round(6)
+      rescue
+        nil
+      end
+
+      def self.lookup_rate(model)
+        # User-configured override wins. Match case-insensitively on the model key.
+        overrides = RailsErrorDashboard.configuration.llm_pricing_overrides || {}
+        normalized = model.downcase
+
+        overrides.each do |key, rate|
+          return symbolize_rate(rate) if key.to_s.downcase == normalized
+        end
+
+        PRICES.each do |key, rate|
+          return rate if key.downcase == normalized
+        end
+
+        nil
+      end
+
+      def self.symbolize_rate(rate)
+        return nil unless rate.is_a?(Hash)
+        {
+          input: rate[:input] || rate["input"],
+          output: rate[:output] || rate["output"]
+        }
+      end
+
+      private_class_method :lookup_rate, :symbolize_rate
+    end
+  end
+end

data/lib/rails_error_dashboard/services/llm_summary.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Services
+    # Pure algorithm: Roll up LLM breadcrumbs into a per-error summary.
+    #
+    # Operates on already-captured breadcrumb data at display time only —
+    # zero runtime cost. Same pattern as CacheAnalyzer / NplusOneDetector.
+    #
+    # NOTE on string coercion: BreadcrumbCollector#truncate_metadata stringifies
+    # every metadata value (input_tokens "42", cost_usd "0.0003", etc.). This
+    # service does the `.to_i` / `.to_f` itself so callers don't have to.
+    #
+    # @example
+    #   RailsErrorDashboard::Services::LlmSummary.call(breadcrumbs)
+    #   # => { total_calls: 3, total_tool_calls: 2, total_input_tokens: 1450,
+    #   #      total_output_tokens: 220, total_tokens: 1670,
+    #   #      total_cost_usd: 0.00821, error_count: 1, total_duration_ms: 4321.5,
+    #   #      providers: ["anthropic", "openai"],
+    #   #      by_model: [ { provider: "openai", model: "gpt-4o-mini",
+    #   #                    calls: 2, tokens: 800, cost_usd: 0.0042 }, ... ] }
+    class LlmSummary
+      # @param breadcrumbs [Array<Hash>] Parsed breadcrumb array
+      # @return [Hash, nil] Summary hash, or nil if no LLM breadcrumbs present
+      def self.call(breadcrumbs)
+        return nil unless breadcrumbs.is_a?(Array)
+
+        llm_crumbs = breadcrumbs.select { |c| c.is_a?(Hash) && c["c"] == "llm" }
+        tool_crumbs = breadcrumbs.select { |c| c.is_a?(Hash) && c["c"] == "llm_tool" }
+        return nil if llm_crumbs.empty? && tool_crumbs.empty?
+
+        total_input = 0
+        total_output = 0
+        total_cost = 0.0
+        total_duration = 0.0
+        error_count = 0
+        providers = {}
+        by_model = {}
+
+        llm_crumbs.each do |crumb|
+          meta = crumb["meta"].is_a?(Hash) ? crumb["meta"] : {}
+          provider = meta["provider"].to_s
+          model = meta["model"].to_s
+          input = meta["input_tokens"].to_i
+          output = meta["output_tokens"].to_i
+          cost = meta["cost_usd"].to_f
+          duration = crumb["d"].to_f
+          status = meta["status"].to_s
+
+          total_input += input
+          total_output += output
+          total_cost += cost
+          total_duration += duration
+          error_count += 1 unless status == "success" || status == ""
+
+          providers[provider] = true unless provider.empty?
+
+          key = [ provider, model ]
+          by_model[key] ||= { provider: provider, model: model, calls: 0, tokens: 0, cost_usd: 0.0 }
+          by_model[key][:calls] += 1
+          by_model[key][:tokens] += input + output
+          by_model[key][:cost_usd] += cost
+        end
+
+        # Tool calls also contribute to duration (visible request impact)
+        tool_crumbs.each do |crumb|
+          total_duration += crumb["d"].to_f
+        end
+
+        {
+          total_calls: llm_crumbs.size,
+          total_tool_calls: tool_crumbs.size,
+          total_input_tokens: total_input,
+          total_output_tokens: total_output,
+          total_tokens: total_input + total_output,
+          total_cost_usd: total_cost.round(6),
+          error_count: error_count,
+          total_duration_ms: total_duration.round(1),
+          providers: providers.keys.sort,
+          by_model: by_model.values.sort_by { |row| -row[:calls] }.map { |row|
+            row[:cost_usd] = row[:cost_usd].round(6)
+            row
+          }
+        }
+      rescue => e
+        RailsErrorDashboard::Logger.debug("[RailsErrorDashboard] LlmSummary.call failed: #{e.message}")
+        nil
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/services/markdown_error_formatter.rb

@@ -15,6 +15,7 @@ module RailsErrorDashboard
       MAX_BACKTRACE_LINES = 15
       MAX_BREADCRUMBS = 10
       MAX_VARIABLES = 10
+      MAX_LLM_CALL_ROWS = 10
 
       # @param error [ErrorLog] An error log record
       # @param related_errors [Array] Related error results with :error and :similarity
@@ -41,6 +42,7 @@ module RailsErrorDashboard
         sections << local_variables_section
         sections << instance_variables_section
         sections << request_context_section
+        sections << llm_calls_section
         sections << breadcrumbs_section
         sections << environment_section
         sections << system_health_section
@@ -198,6 +200,91 @@ module RailsErrorDashboard
         "## Request Context\n\n#{items.join("\n")}"
       end
 
+      def llm_calls_section
+        raw = @error.breadcrumbs
+        return nil if raw.blank?
+
+        crumbs = parse_json(raw)
+        return nil unless crumbs.is_a?(Array) && crumbs.any?
+
+        summary = LlmSummary.call(crumbs)
+        return nil unless summary
+
+        llm_crumbs = crumbs.select { |c| c.is_a?(Hash) && (c["c"] == "llm" || c["c"] == "llm_tool") }
+        return nil if llm_crumbs.empty?
+
+        # One-line totals: tokens / cost / duration / errors
+        totals_parts = []
+        totals_parts << "#{summary[:total_calls]} call#{summary[:total_calls] == 1 ? "" : "s"}"
+        totals_parts << "#{summary[:total_tool_calls]} tool call#{summary[:total_tool_calls] == 1 ? "" : "s"}" if summary[:total_tool_calls] > 0
+        totals_parts << "#{summary[:total_tokens]} tokens (in:#{summary[:total_input_tokens]}/out:#{summary[:total_output_tokens]})" if summary[:total_tokens] > 0
+        totals_parts << "$#{format("%.6f", summary[:total_cost_usd])}" if summary[:total_cost_usd] > 0
+        totals_parts << "#{summary[:total_duration_ms]}ms total"
+        totals_parts << "**#{summary[:error_count]} error#{summary[:error_count] == 1 ? "" : "s"}**" if summary[:error_count] > 0
+
+        out = [ "## LLM Calls", "", totals_parts.join(" · ") ]
+
+        if summary[:by_model].any?
+          out << ""
+          out << "**By Model**"
+          out << ""
+          out << "| Provider | Model | Calls | Tokens | Cost |"
+          out << "|----------|-------|-------|--------|------|"
+          summary[:by_model].each do |row|
+            cost_cell = row[:cost_usd] > 0 ? "$#{format("%.6f", row[:cost_usd])}" : "—"
+            out << "| #{row[:provider].presence || "—"} | #{row[:model].presence || "—"} | #{row[:calls]} | #{row[:tokens]} | #{cost_cell} |"
+          end
+        end
+
+        # Per-call detail table — last N events (chat + tool, in order)
+        recent = llm_crumbs.last(MAX_LLM_CALL_ROWS)
+        out << ""
+        out << "**Calls (last #{recent.size})**"
+        out << ""
+        out << "| Time | Type | Provider/Model | Status | Tokens | Cost | Duration |"
+        out << "|------|------|----------------|--------|--------|------|----------|"
+        recent.each do |c|
+          meta = c["meta"].is_a?(Hash) ? c["meta"] : {}
+          time = c["t"] ? Time.at(c["t"] / 1000.0).utc.strftime("%H:%M:%S.%L") : "—"
+          type = c["c"] == "llm_tool" ? "tool" : "chat"
+          provider_model = if c["c"] == "llm_tool"
+            "tool: #{truncate_value(meta["tool_name"], 40)}"
+          else
+            [ meta["provider"].presence, meta["model"].presence ].compact.join("/").presence || "—"
+          end
+          status = meta["status"].presence || "—"
+          tokens = if meta["input_tokens"].present? || meta["output_tokens"].present?
+            "#{meta["input_tokens"] || "?"}/#{meta["output_tokens"] || "?"}"
+          else
+            "—"
+          end
+          cost = meta["cost_usd"].present? && meta["cost_usd"].to_f > 0 ? "$#{format("%.6f", meta["cost_usd"].to_f)}" : "—"
+          duration = c["d"] ? "#{c["d"]}ms" : "—"
+          out << "| #{time} | #{type} | #{truncate_value(provider_model, 40)} | #{status} | #{tokens} | #{cost} | #{duration} |"
+        end
+
+        # Surface error messages so the LLM consumer sees them inline rather than
+        # having to cross-reference the breadcrumb stream below.
+        error_lines = llm_crumbs.select { |c|
+          c["c"] == "llm" && c["meta"].is_a?(Hash) && c["meta"]["status"] && c["meta"]["status"] != "success"
+        }
+        if error_lines.any?
+          out << ""
+          out << "**Failures**"
+          out << ""
+          error_lines.each do |c|
+            meta = c["meta"]
+            label = [ meta["provider"], meta["model"] ].compact.join("/")
+            err_class = meta["error_class"].presence
+            err_msg = meta["error_message"].presence
+            detail = [ err_class, err_msg ].compact.join(": ")
+            out << "- `#{label}` — #{meta["status"]}#{detail.empty? ? "" : ": #{detail}"}"
+          end
+        end
+
+        out.join("\n")
+      end
+
       def breadcrumbs_section
         raw = @error.breadcrumbs
         return nil if raw.blank?