rails_error_dashboard 0.6.4 → 0.7.0

data/lib/rails_error_dashboard/integrations/llm_middleware.rb

@@ -0,0 +1,276 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Integrations
+    # Faraday middleware that captures LLM calls to OpenAI and Anthropic APIs
+    # as breadcrumbs. The Tier 2 path — for hosts using `ruby-openai` or
+    # `anthropic-sdk-ruby` directly without OpenTelemetry instrumentation.
+    #
+    # Install in the host app:
+    #
+    #   # Anthropic SDK
+    #   Anthropic::Client.new do |f|
+    #     f.use RailsErrorDashboard::Integrations::LlmMiddleware
+    #   end
+    #
+    #   # ruby-openai
+    #   OpenAI::Client.new do |f|
+    #     f.use RailsErrorDashboard::Integrations::LlmMiddleware
+    #   end
+    #
+    # IMPORTANT — does NOT subclass ::Faraday::Middleware. Doing so would
+    # NameError at file-load time on hosts without Faraday. Faraday accepts
+    # any object that responds to `#call(env)` and is initialized with `app`.
+    # Hosts that don't use OpenAI/Anthropic SDKs simply won't reference this
+    # class and never load the constant.
+    #
+    # HOST APP SAFETY:
+    # - Wraps the upstream call in rescue, but ALWAYS re-raises (we are in
+    #   the host's request path — swallowing would break their app logic)
+    # - Our own bookkeeping (response parsing, breadcrumb emission) is wrapped
+    #   separately in rescue StandardError => nil
+    # - No work happens unless enable_llm_observability AND enable_breadcrumbs
+    # - Non-LLM URLs (anything but api.openai.com / api.anthropic.com) skip
+    #   straight through with one host-string comparison
+    # - Streaming responses (SSE) skipped — token counts only available in
+    #   the final stream event, which we'd need to buffer to read
+    class LlmMiddleware
+      OPENAI_HOSTS    = [ "api.openai.com" ].freeze
+      ANTHROPIC_HOSTS = [ "api.anthropic.com" ].freeze
+
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        return @app.call(env) unless RailsErrorDashboard.configuration.enable_llm_observability
+        return @app.call(env) unless RailsErrorDashboard.configuration.enable_breadcrumbs
+
+        provider = detect_provider(env)
+        return @app.call(env) unless provider
+
+        request_body = safe_parse_body(env.body)
+        model        = request_body.is_a?(Hash) ? request_body["model"] : nil
+        started_at   = monotonic_ms
+
+        response = nil
+        upstream_error = nil
+        begin
+          response = @app.call(env)
+        rescue StandardError => e
+          upstream_error = e
+          raise
+        ensure
+          # Record the breadcrumb whether the call succeeded, returned an HTTP
+          # error, or raised mid-flight. NEVER raise from this block — the
+          # host's app.call has either returned or is propagating an exception
+          # via `raise` above, and we must not interfere with either path.
+          begin
+            duration_ms = (monotonic_ms - started_at).round(2)
+            emit_breadcrumb(provider, model, request_body, response, upstream_error, duration_ms)
+          rescue StandardError => e
+            RailsErrorDashboard::Logger.debug("[RailsErrorDashboard] LlmMiddleware.emit failed: #{e.message}")
+          end
+        end
+
+        response
+      end
+
+      private
+
+      def detect_provider(env)
+        host = env.url&.host
+        return nil unless host
+        return "openai"    if OPENAI_HOSTS.include?(host)
+        return "anthropic" if ANTHROPIC_HOSTS.include?(host)
+        nil
+      end
+
+      def safe_parse_body(body)
+        return body if body.is_a?(Hash)
+        return {} if body.nil? || (body.respond_to?(:empty?) && body.empty?)
+        JSON.parse(body.to_s)
+      rescue StandardError
+        {}
+      end
+
+      def streaming_response?(response)
+        return false unless response
+        ct = response.respond_to?(:headers) ? response.headers&.[]("content-type") : nil
+        ct.is_a?(String) && ct.include?("text/event-stream")
+      end
+
+      def emit_breadcrumb(provider, model, request_body, response, upstream_error, duration_ms)
+        if upstream_error
+          event = build_error_event(provider, model, upstream_error, duration_ms)
+          add_crumb(event)
+          return
+        end
+
+        # Streaming — token counts aren't available without buffering the
+        # stream, which would defeat the SDK's streaming behavior. Skip for
+        # v0.7.0; a future release can add an SSE parser if demand warrants.
+        if streaming_response?(response)
+          RailsErrorDashboard::Logger.debug("[RailsErrorDashboard] LlmMiddleware skipping streaming response (#{provider})")
+          return
+        end
+
+        status      = response.respond_to?(:status) ? response.status : nil
+        body        = parse_response_body(response)
+
+        if status && status >= 400
+          add_crumb(build_http_error_event(provider, model, status, body, duration_ms))
+          return
+        end
+
+        add_crumb(build_success_event(provider, model, request_body, body, duration_ms))
+      end
+
+      def parse_response_body(response)
+        body = response.respond_to?(:body) ? response.body : nil
+        return body if body.is_a?(Hash)
+        return {} if body.nil? || (body.respond_to?(:empty?) && body.empty?)
+        JSON.parse(body.to_s)
+      rescue StandardError
+        {}
+      end
+
+      def build_success_event(provider, request_model, request_body, response_body, duration_ms)
+        response_model = response_body.is_a?(Hash) ? response_body["model"] : nil
+        model = response_model || request_model
+
+        input_tokens, output_tokens = extract_tokens(provider, response_body)
+        tool_calls_requested        = extract_tool_calls(provider, response_body)
+
+        cost = Services::LlmCostEstimator.estimate(
+          provider: provider,
+          model: model,
+          input_tokens: input_tokens,
+          output_tokens: output_tokens
+        )
+
+        ValueObjects::LlmCallEvent.new(
+          provider: provider,
+          model: model || "unknown",
+          status: :success,
+          input_tokens: input_tokens,
+          output_tokens: output_tokens,
+          duration_ms: duration_ms,
+          cost_usd_estimate: cost,
+          tool_arguments: tool_calls_metadata(tool_calls_requested)
+        )
+      end
+
+      def build_http_error_event(provider, model, status, body, duration_ms)
+        err_class = "HTTP #{status}"
+        err_msg   = extract_error_message(body)
+
+        ValueObjects::LlmCallEvent.new(
+          provider: provider,
+          model: model || "unknown",
+          status: :error,
+          duration_ms: duration_ms,
+          error_class: err_class,
+          error_message: err_msg
+        )
+      end
+
+      def build_error_event(provider, model, exception, duration_ms)
+        status = exception_status(exception)
+        ValueObjects::LlmCallEvent.new(
+          provider: provider,
+          model: model || "unknown",
+          status: status,
+          duration_ms: duration_ms,
+          error_class: exception.class.name,
+          error_message: exception.message
+        )
+      end
+
+      def exception_status(exception)
+        klass = exception.class.name.to_s
+        return :timeout if klass.include?("Timeout") || klass.include?("TimedOut")
+        :error
+      end
+
+      # @return [Array(Integer|nil, Integer|nil)] (input_tokens, output_tokens)
+      def extract_tokens(provider, body)
+        return [ nil, nil ] unless body.is_a?(Hash)
+        usage = body["usage"]
+        return [ nil, nil ] unless usage.is_a?(Hash)
+
+        case provider
+        when "openai"
+          [ usage["prompt_tokens"], usage["completion_tokens"] ]
+        when "anthropic"
+          [ usage["input_tokens"], usage["output_tokens"] ]
+        else
+          [ nil, nil ]
+        end
+      end
+
+      # Returns an Array of tool-call descriptors: [{ name: "...", id: "..." }, ...]
+      # Empty when the model didn't request any tools.
+      def extract_tool_calls(provider, body)
+        return [] unless body.is_a?(Hash)
+
+        case provider
+        when "openai"
+          choices = body["choices"]
+          return [] unless choices.is_a?(Array) && choices.any?
+          tool_calls = choices.first.dig("message", "tool_calls")
+          return [] unless tool_calls.is_a?(Array)
+          tool_calls.filter_map do |tc|
+            next unless tc.is_a?(Hash)
+            name = tc.dig("function", "name")
+            name ? { name: name, id: tc["id"] } : nil
+          end
+        when "anthropic"
+          content = body["content"]
+          return [] unless content.is_a?(Array)
+          content.filter_map do |c|
+            next unless c.is_a?(Hash) && c["type"] == "tool_use"
+            { name: c["name"], id: c["id"] }
+          end
+        else
+          []
+        end
+      end
+
+      # Compact summary of tool calls — packed into the
+      # `tool_arguments` field on LlmCallEvent so it lands in breadcrumb
+      # metadata under `:tool_arguments`. (We reuse the existing slot rather
+      # than adding a new field for v0.7.0; UI in 4.1 reads it back.)
+      # Returns nil when no tools were requested so the field omits from JSON.
+      def tool_calls_metadata(tool_calls)
+        return nil if tool_calls.nil? || tool_calls.empty?
+        names = tool_calls.first(3).map { |tc| tc[:name] }.compact
+        suffix = tool_calls.size > 3 ? "+#{tool_calls.size - 3} more" : nil
+        [ "tools:#{tool_calls.size}", names.join(","), suffix ].compact.join(" ")
+      end
+
+      def extract_error_message(body)
+        return nil unless body.is_a?(Hash)
+        # OpenAI: { "error": { "message": "...", "type": "...", "code": "..." } }
+        # Anthropic: { "error": { "type": "...", "message": "..." }, "type": "error" }
+        err = body["error"]
+        return nil unless err.is_a?(Hash)
+        err["message"] || err["type"]
+      end
+
+      def add_crumb(event)
+        category = event.tool_call? ? "llm_tool" : "llm"
+        Services::BreadcrumbCollector.add(
+          category,
+          event.to_breadcrumb_message,
+          duration_ms: event.duration_ms,
+          metadata: event.to_breadcrumb_metadata
+        )
+      end
+
+      def monotonic_ms
+        Process.clock_gettime(Process::CLOCK_MONOTONIC) * 1000.0
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/integrations/llm_span_processor.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Integrations
+    # OpenTelemetry SpanProcessor that maps GenAI semantic-convention spans
+    # into LLM breadcrumbs. Registered with `OpenTelemetry.tracer_provider`
+    # when the host app already runs OTel (ruby_llm, thoughtbot/instrumentation,
+    # etc. all emit GenAI spans automatically).
+    #
+    # IMPORTANT — does NOT subclass ::OpenTelemetry::SDK::Trace::SpanProcessor.
+    # That would NameError at file-load time on hosts without the SDK. Ruby's
+    # OTel SDK accepts any duck-typed processor — name + arity is the contract.
+    #
+    # Reads attribute keys per the GenAI semconv (current + deprecated aliases).
+    # Spec: https://opentelemetry.io/docs/specs/semconv/gen-ai/
+    #
+    # HOST APP SAFETY:
+    # - on_finish wraps the entire body in rescue StandardError => nil
+    # - No work happens unless enable_llm_observability AND enable_breadcrumbs
+    # - Non-GenAI spans return immediately (cheapest possible path)
+    # - Never raises, never blocks the tracer pipeline
+    class LlmSpanProcessor
+      class << self
+        # Idempotently register a single shared LlmSpanProcessor instance with
+        # the host's OpenTelemetry tracer provider. Called from Engine
+        # `after_initialize` when `enable_llm_observability` is on.
+        #
+        # Returns false (and does nothing) when:
+        # - OTel SDK isn't loaded (`Integrations::OTel.available?` is false)
+        # - `enable_llm_observability` is off
+        # - The active tracer provider is the default `ProxyTracerProvider`
+        #   (SDK loaded but `OpenTelemetry::SDK.configure` never called) —
+        #   detected by absence of `add_span_processor`
+        # - Already registered in this process (Spring reload safety)
+        # - `add_span_processor` raises (host app safety — never crash boot)
+        #
+        # @return [Boolean] true if a processor was newly registered, false otherwise
+        def register!
+          return false if @registered
+          return false unless RailsErrorDashboard.configuration.enable_llm_observability
+          return false unless OTel.available?
+
+          provider = ::OpenTelemetry.tracer_provider
+          return false unless provider.respond_to?(:add_span_processor)
+
+          provider.add_span_processor(new)
+          @registered = true
+          true
+        rescue StandardError => e
+          RailsErrorDashboard::Logger.debug("[RailsErrorDashboard] LlmSpanProcessor.register! failed: #{e.message}")
+          false
+        end
+
+        # Test hook — clear the registered flag so re-registration is possible
+        # in a fresh spec example. Does NOT remove the processor from the
+        # tracer provider (OTel SDK offers no symmetric `remove_span_processor`).
+        def reset!
+          @registered = false
+        end
+
+        # @return [Boolean]
+        def registered?
+          @registered == true
+        end
+      end
+
+      # Attribute keys — current GenAI semconv, with deprecated aliases.
+      PROVIDER_KEYS   = [ "gen_ai.provider.name", "gen_ai.system" ].freeze
+      MODEL_KEYS      = [ "gen_ai.response.model", "gen_ai.request.model" ].freeze
+      INPUT_TOKEN_KEYS  = [ "gen_ai.usage.input_tokens", "gen_ai.usage.prompt_tokens" ].freeze
+      OUTPUT_TOKEN_KEYS = [ "gen_ai.usage.output_tokens", "gen_ai.usage.completion_tokens" ].freeze
+      TOOL_NAME_KEY     = "gen_ai.tool.name"
+      OPERATION_KEY     = "gen_ai.operation.name"
+      ERROR_TYPE_KEY    = "error.type"
+
+      # Required SpanProcessor interface — no-op. We only act when the span
+      # is fully populated (attributes/timestamps/status), which is on_finish.
+      def on_start(_span, _parent_context)
+        nil
+      end
+
+      # Required SpanProcessor interface. Must never raise.
+      def on_finish(span)
+        return unless RailsErrorDashboard.configuration.enable_llm_observability
+        return unless RailsErrorDashboard.configuration.enable_breadcrumbs
+
+        attrs = safe_attributes(span)
+        return if attrs.empty?
+        return unless gen_ai_span?(attrs)
+
+        event   = build_event(span, attrs)
+        category = event.tool_call? ? "llm_tool" : "llm"
+
+        Services::BreadcrumbCollector.add(
+          category,
+          event.to_breadcrumb_message,
+          duration_ms: event.duration_ms,
+          metadata: event.to_breadcrumb_metadata
+        )
+      rescue StandardError => e
+        RailsErrorDashboard::Logger.debug("[RailsErrorDashboard] LlmSpanProcessor.on_finish failed: #{e.message}")
+        nil
+      end
+
+      # OTel SDK Export::SUCCESS == 0. Hardcoded so this file loads without OTel.
+      def force_flush(timeout: nil)
+        0
+      end
+
+      def shutdown(timeout: nil)
+        0
+      end
+
+      private
+
+      def safe_attributes(span)
+        attrs = span.attributes
+        attrs.is_a?(Hash) ? attrs : {}
+      rescue StandardError
+        {}
+      end
+
+      # Cheap pre-filter — only inspect spans that actually carry GenAI semconv.
+      def gen_ai_span?(attrs)
+        PROVIDER_KEYS.any? { |k| attrs.key?(k) } ||
+          MODEL_KEYS.any? { |k| attrs.key?(k) } ||
+          attrs.key?(OPERATION_KEY) ||
+          attrs.key?(TOOL_NAME_KEY)
+      end
+
+      def build_event(span, attrs)
+        provider      = first_attr(attrs, PROVIDER_KEYS)
+        model         = first_attr(attrs, MODEL_KEYS)
+        input_tokens  = first_attr(attrs, INPUT_TOKEN_KEYS)
+        output_tokens = first_attr(attrs, OUTPUT_TOKEN_KEYS)
+        tool_name     = attrs[TOOL_NAME_KEY] || (attrs[OPERATION_KEY] == "execute_tool" ? attrs[OPERATION_KEY] : nil)
+        error_type    = attrs[ERROR_TYPE_KEY]
+
+        status = error_type ? :error : :success
+        duration_ms = compute_duration_ms(span)
+
+        cost = nil
+        if status == :success && tool_name.nil? && model
+          cost = Services::LlmCostEstimator.estimate(
+            provider: provider,
+            model: model,
+            input_tokens: input_tokens,
+            output_tokens: output_tokens
+          )
+        end
+
+        ValueObjects::LlmCallEvent.new(
+          provider: provider || "unknown",
+          model: model || "unknown",
+          status: status,
+          input_tokens: input_tokens,
+          output_tokens: output_tokens,
+          duration_ms: duration_ms,
+          error_class: error_type,
+          tool_name: tool_name,
+          cost_usd_estimate: cost
+        )
+      end
+
+      def first_attr(attrs, keys)
+        keys.each { |k| return attrs[k] if attrs.key?(k) }
+        nil
+      end
+
+      # OTel timestamps are nanoseconds since epoch. Convert to ms; guard nils.
+      def compute_duration_ms(span)
+        start_ns = span.start_timestamp
+        end_ns   = span.end_timestamp
+        return nil unless start_ns && end_ns
+        ((end_ns - start_ns) / 1_000_000.0).round(2)
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/integrations/o_tel.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Integrations
+    # Detection shim for the OpenTelemetry SDK. The LLM-observability feature
+    # registers a SpanProcessor against `OpenTelemetry.tracer_provider` when
+    # the host app already runs OTel — for ruby_llm and thoughtbot users
+    # this is the zero-config path. When OTel is absent, we silently skip
+    # the SpanProcessor (the Faraday middleware path still works).
+    #
+    # `opentelemetry-sdk` is an OPTIONAL dependency. This module must never
+    # raise, never require the gem itself, and never assume the host has it.
+    module OTel
+      class << self
+        # Returns true when the OpenTelemetry SDK is loaded and the
+        # SpanProcessor base class is reachable (Task 2.2 subclasses it).
+        # Memoized — host apps don't dynamically load gems mid-process.
+        # Rescues any unexpected error to a hard false: a broken partial
+        # install must never block a request in the host app.
+        # @return [Boolean]
+        def available?
+          return @available unless @available.nil?
+          @available = detect
+        rescue StandardError
+          @available = false
+        end
+
+        # Test hook — clears the memoized result so specs can flip
+        # OpenTelemetry constants in/out between examples.
+        def reset!
+          @available = nil
+        end
+
+        private
+
+        def detect
+          return false unless defined?(::OpenTelemetry)
+          return false unless defined?(::OpenTelemetry::SDK)
+          return false unless defined?(::OpenTelemetry::SDK::Trace::SpanProcessor)
+          true
+        end
+      end
+    end
+  end
+end