rails_error_dashboard 0.12.1 → 0.13.0

data/lib/rails_error_dashboard/services/error_broadcaster.rb

@@ -15,26 +15,71 @@ module RailsErrorDashboard
     # are NOT available there. We render via the engine's own controller renderer
     # and pass pre-rendered HTML to the broadcast to ensure route helpers work.
     class ErrorBroadcaster
+      # Minimum seconds between two stats broadcasts on one stream, per process.
+      # The row broadcasts are cheap (one partial); the stats payload is a full
+      # DashboardStats computation, and a capture runs it from inside the host
+      # app's request thread. Leading edge: the first event in a window
+      # broadcasts, later ones are dropped and corrected by the next window or
+      # the next page load.
+      STATS_BROADCAST_INTERVAL = 5
+
+      # Stream names, in ONE place for the view (turbo_stream_from) and for this
+      # class. Three kinds, because a Turbo subscription is all-or-nothing per
+      # stream and the index needs them separately: a view filtered by platform
+      # wants row REPLACES (harmless: they only touch rows already on the page)
+      # but not PREPENDS (a new row may not match its filter).
+      #
+      # Every kind exists globally and per application ("error_list_app_7"), so
+      # a view filtered to one application never receives another's rows.
+      STREAMS = { list: "error_list", updates: "error_updates", stats: "error_stats" }.freeze
+
+      # Column visibility (which optional cells a row has) used to be two
+      # SELECT DISTINCT scans per broadcast. It changes about never.
+      COLUMN_VISIBILITY_TTL = 60.seconds
+
+      # One throttle entry per stats stream, i.e. per application. Bounded so a
+      # host with thousands of applications cannot grow it without limit.
+      MAX_THROTTLED_STREAMS = 200
+
+      THROTTLE_MUTEX = Mutex.new
+      private_constant :THROTTLE_MUTEX
+
+      # @param kind [Symbol] :list (new rows), :updates (row replaces) or :stats
+      # @param application_id [Integer, String, nil] nil/blank for the global stream
+      # @return [String]
+      def self.stream_name(kind, application_id = nil)
+        base = STREAMS.fetch(kind)
+        return base if application_id.blank?
+
+        # to_i: the id reaches here from a request parameter in the view.
+        "#{base}_app_#{application_id.to_s[/\A\d+\z/].to_i}"
+      end
+
+      # The streams a dashboard index page subscribes to.
+      # @param application_id [Integer, String, nil] the page's application filter
+      # @param filtered [Boolean] any OTHER filter, a sort or a later page is active,
+      #   so a new row cannot simply be put on top of the list
+      # @return [Array<String>]
+      def self.streams_for_view(application_id: nil, filtered: false)
+        kinds = filtered ? %i[updates stats] : %i[list updates stats]
+        kinds.map { |kind| stream_name(kind, application_id) }
+      end
+
       # Broadcast a new error (prepend to error list + refresh stats)
       # @param error_log [ErrorLog] The newly created error
       def self.broadcast_new(error_log)
         return unless error_log
         return unless available?
+        return unless calm?
 
-        platforms = ErrorLog.distinct.pluck(:platform).compact
-        show_platform = platforms.size > 1
-        show_environment = ErrorLog.column_names.include?("environment") &&
-                           ErrorLog.distinct.pluck(:environment).compact.size > 1
-
-        html = render_partial("rails_error_dashboard/errors/error_row",
-          error: error_log, show_platform: show_platform, show_environment: show_environment)
-
-        Turbo::StreamsChannel.broadcast_prepend_to(
-          "error_list",
-          target: "error_list",
-          html: html
-        )
-        broadcast_stats
+        each_row_html(error_log) do |application_id, html|
+          Turbo::StreamsChannel.broadcast_prepend_to(
+            stream_name(:list, application_id),
+            target: "error_list",
+            html: html
+          )
+        end
+        broadcast_stats(error_log.application_id)
       rescue => e
         Rails.logger.error("[RailsErrorDashboard] Failed to broadcast new error: #{e.class} - #{e.message}")
         Rails.logger.debug("[RailsErrorDashboard] Backtrace: #{e.backtrace&.first(3)&.join("\n")}")
@@ -45,37 +90,45 @@ module RailsErrorDashboard
       def self.broadcast_update(error_log)
         return unless error_log
         return unless available?
+        return unless calm?
 
-        platforms = ErrorLog.distinct.pluck(:platform).compact
-        show_platform = platforms.size > 1
-        show_environment = ErrorLog.column_names.include?("environment") &&
-                           ErrorLog.distinct.pluck(:environment).compact.size > 1
-
-        html = render_partial("rails_error_dashboard/errors/error_row",
-          error: error_log, show_platform: show_platform, show_environment: show_environment)
-
-        Turbo::StreamsChannel.broadcast_replace_to(
-          "error_list",
-          target: "error_#{error_log.id}",
-          html: html
-        )
-        broadcast_stats
+        each_row_html(error_log) do |application_id, html|
+          Turbo::StreamsChannel.broadcast_replace_to(
+            stream_name(:updates, application_id),
+            target: "error_#{error_log.id}",
+            html: html
+          )
+        end
+        broadcast_stats(error_log.application_id)
       rescue => e
         Rails.logger.error("[RailsErrorDashboard] Failed to broadcast error update: #{e.class} - #{e.message}")
         Rails.logger.debug("[RailsErrorDashboard] Backtrace: #{e.backtrace&.first(3)&.join("\n")}")
       end
 
-      # Broadcast stats refresh
-      def self.broadcast_stats
+      # Broadcast a stats refresh: all-application stats to the global stream,
+      # application-scoped stats to that application's stream.
+      #
+      # ONE event computes at most ONE payload. Each stream is throttled on its
+      # own (STATS_BROADCAST_INTERVAL); when both are due, the one that has
+      # waited longest goes and the other is served by the next event. Computing
+      # both would double the most expensive thing a capture does, and a
+      # fixed order would starve the second stream whenever events are rare.
+      #
+      # @param application_id [Integer, nil] the application of the event, if any
+      def self.broadcast_stats(application_id = nil)
         return unless available?
+        return unless calm?
 
-        stats = Queries::DashboardStats.call
+        scope = claim_stats_window!([ nil, application_id.presence ].uniq)
+        return if scope == :none
+
+        stats = scope ? Queries::DashboardStats.call(application_id: scope) : Queries::DashboardStats.call
         return unless stats.is_a?(Hash) && stats.present?
 
         html = render_partial("rails_error_dashboard/errors/stats", stats: stats)
 
         Turbo::StreamsChannel.broadcast_replace_to(
-          "error_list",
+          stream_name(:stats, scope),
           target: "dashboard_stats",
           html: html
         )
@@ -84,6 +137,101 @@ module RailsErrorDashboard
         Rails.logger.debug("[RailsErrorDashboard] Backtrace: #{e.backtrace&.first(3)&.join("\n")}")
       end
 
+      # Yields [application_id_or_nil, row_html] once per row stream. The global
+      # table has an Application column when there is more than one application;
+      # an application's own table never does, so the two need different cells.
+      # Rendered once when they come out the same.
+      def self.each_row_html(error_log)
+        render = lambda do |visibility, show_application|
+          render_partial("rails_error_dashboard/errors/error_row",
+            error: error_log, show_platform: visibility[:platform],
+            show_environment: visibility[:environment], show_application: show_application)
+        end
+
+        global = column_visibility(nil)
+        yield nil, render.call(global, global[:application])
+
+        application_id = error_log.application_id
+        return if application_id.blank?
+
+        yield application_id, render.call(column_visibility(application_id), false)
+      end
+
+      # Which optional columns the index table shows, for the global view or for
+      # one application -- the same rule the index itself applies. Cached for
+      # COLUMN_VISIBILITY_TTL and keyed on the cache generation, so a user
+      # action that could change it (a batch delete) refreshes it at once.
+      def self.column_visibility(application_id)
+        key = [ "red/columns", AnalyticsCacheManager.generation, application_id || "all" ].join("/")
+
+        Rails.cache.fetch(key, expires_in: COLUMN_VISIBILITY_TTL) do
+          scope = application_id ? ErrorLog.where(application_id: application_id) : ErrorLog.all
+          {
+            platform: scope.distinct.pluck(:platform).compact.size > 1,
+            environment: ErrorLog.column_names.include?("environment") &&
+              scope.distinct.pluck(:environment).compact.size > 1,
+            application: application_id.nil? && Application.count > 1
+          }
+        end
+      end
+
+      # Live updates are a convenience; during a storm they are load. While the
+      # breaker is anything but :closed nothing is broadcast -- the page catches
+      # up on its next load. Fails towards broadcasting: Gate.state itself
+      # answers :closed when storm protection is off or unreadable.
+      def self.calm?
+        StormProtection::Gate.state == :closed
+      rescue StandardError
+        true
+      end
+
+      # Picks which stats scope this event may broadcast, and claims its window.
+      #
+      # @param scopes [Array<Integer, nil>] candidate scopes (nil = all applications)
+      # @return [Integer, nil, :none] the claimed scope, or :none when every
+      #   candidate is still inside its window
+      #
+      # The claim is taken BEFORE the stats are computed, so concurrent captures
+      # cannot all decide to compute at once, and a computation that fails does
+      # not get retried by every event that follows it.
+      def self.claim_stats_window!(scopes = [ nil ])
+        THROTTLE_MUTEX.synchronize do
+          @stats_claimed_at ||= {}
+          now = monotonic_now
+
+          due = scopes.select do |scope|
+            last = @stats_claimed_at[scope]
+            last.nil? || (now - last) >= STATS_BROADCAST_INTERVAL
+          end
+          return :none if due.empty?
+
+          # Longest-waiting first; never-claimed counts as waiting forever.
+          # min_by is stable, so the global stream wins a tie.
+          chosen = due.min_by { |scope| @stats_claimed_at[scope] || -Float::INFINITY }
+          @stats_claimed_at[chosen] = now
+
+          if @stats_claimed_at.size > MAX_THROTTLED_STREAMS
+            stalest = @stats_claimed_at.min_by { |_scope, at| at }.first
+            @stats_claimed_at.delete(stalest)
+          end
+
+          chosen
+        end
+      end
+
+      def self.throttled_stream_count
+        THROTTLE_MUTEX.synchronize { (@stats_claimed_at || {}).size }
+      end
+
+      def self.monotonic_now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      # For specs and for a forked worker that wants a clean slate.
+      def self.reset_throttle!
+        THROTTLE_MUTEX.synchronize { @stats_claimed_at = {} }
+      end
+
       # Render a partial using the engine's controller renderer.
       # This ensures engine route helpers (error_path, etc.) are available,
       # unlike Turbo's default ApplicationController.render which uses the host app's context.

data/lib/rails_error_dashboard/services/error_hash_generator.rb

@@ -121,8 +121,13 @@ module RailsErrorDashboard
       # @param message [String, nil] The error message
       # @return [String, nil] Normalized message
       def self.normalize_message(message)
-        message
-          &.[](0, HASH_MESSAGE_LIMIT)
+        # Slice, THEN scrub, then match: every path (sync, async, storm gate)
+        # goes through here, so they all hash the same text. gsub raises on a
+        # string with invalid bytes; a valid message is returned by scrub as-is,
+        # so no existing fingerprint changes.
+        prefix = message&.[](0, HASH_MESSAGE_LIMIT)
+
+        EncodingSanitizer.scrub(prefix)
           &.gsub(/0x[0-9a-f]+/i, "HEX")          # Replace hex addresses (before numbers)
           &.gsub(/#<[^>]+>/, "#<OBJ>")           # Replace object inspections
           &.gsub(/\d+/, "N")                     # Replace numbers
@@ -161,7 +166,8 @@ module RailsErrorDashboard
           !frame.include?("/gems/")
         }
 
-        first_app_frame&.split(":")&.first
+        # split raises on a frame with invalid bytes (a method name can hold any).
+        EncodingSanitizer.scrub(first_app_frame)&.split(":")&.first
       end
 
       # Try custom fingerprint lambda if configured

data/lib/rails_error_dashboard/services/error_notification_dispatcher.rb

@@ -10,6 +10,19 @@ module RailsErrorDashboard
     # @example
     #   ErrorNotificationDispatcher.call(error_log)
     class ErrorNotificationDispatcher
+      # Is any channel configured to the point where .call would enqueue
+      # something? The same conditions as below, without the side effects.
+      # @return [Boolean]
+      def self.any_channel?
+        config = RailsErrorDashboard.configuration
+
+        (config.enable_slack_notifications && config.slack_webhook_url.present?) ||
+          (config.enable_email_notifications && config.notification_email_recipients.present?) ||
+          (config.enable_discord_notifications && config.discord_webhook_url.present?) ||
+          (config.enable_pagerduty_notifications && config.pagerduty_integration_key.present?) ||
+          (config.enable_webhook_notifications && config.webhook_urls.present?) || false
+      end
+
       # @param error_log [ErrorLog] The error to notify about
       def self.call(error_log)
         # OTel: emit a child span around the dispatch so operators can see

data/lib/rails_error_dashboard/services/exception_filter.rb

@@ -10,6 +10,14 @@ module RailsErrorDashboard
     # @example
     #   ExceptionFilter.should_log?(exception) # => true/false
     class ExceptionFilter
+      # Distinct errors remembered for first-seen admission (see sampled_out?).
+      # Eviction only costs one extra admitted sample, so there is no overflow
+      # accounting here.
+      MAX_SEEN = 1_000
+
+      @seen = {}
+      @seen_mutex = Mutex.new
+
       # Check if an exception should be logged (not ignored, not sampled out)
       # @param exception [Exception] The exception to check
       # @return [Boolean] true if the exception should be logged
@@ -48,16 +56,63 @@ module RailsErrorDashboard
       # Critical errors are ALWAYS logged regardless of sampling
       # @param exception [Exception] The exception to check
       # @return [Boolean] true if the exception should be skipped
+      #
+      # The FIRST event of each error in this process is always admitted.
+      # Sampling is a dice roll taken before anything is known about the error,
+      # so at a rate of 0.01 an error that happens three times is usually never
+      # recorded at all. Sampling is there to cut volume; it must not hide that
+      # an error exists. A rate of 0.0 stays a hard off switch.
       def self.sampled_out?(exception)
         sampling_rate = RailsErrorDashboard.configuration.sampling_rate
 
         return false if sampling_rate >= 1.0
         return false if critical?(exception)
         return true if sampling_rate <= 0.0
+        return false if first_sighting?(exception)
 
         rand > sampling_rate
       end
 
+      # True exactly once per process for each (exception class, first
+      # application frame). Coarser than the real fingerprint on purpose: the
+      # fingerprint needs the application id (a query) and this runs before the
+      # storm gate. Two errors raised from one line share a "first".
+      #
+      # Per process, so N workers admit N firsts. Bounded at MAX_SEEN, least
+      # recently seen evicted first. Any failure means "not seen": admit.
+      def self.first_sighting?(exception)
+        key = seen_key(exception)
+
+        @seen_mutex.synchronize do
+          seen_before = !@seen.delete(key).nil?
+          @seen.shift while @seen.size >= MAX_SEEN
+          @seen[key] = true
+          !seen_before
+        end
+      rescue => e
+        RailsErrorDashboard::Logger.debug("[RailsErrorDashboard] ExceptionFilter.first_sighting? failed: #{e.class}: #{e.message}")
+        true
+      end
+
+      # "ClassName|first application frame". The first backtrace line that is
+      # not inside a gem or the Ruby core; the first line when there is no such
+      # frame; nothing when the exception was never raised.
+      def self.seen_key(exception)
+        backtrace = exception.backtrace || []
+        frame = backtrace.find { |line| !line.include?("/gems/") && !line.start_with?("<internal:") } || backtrace.first
+
+        "#{exception.class.name}|#{frame.to_s.sub(/:in .*\z/, "")}"
+      end
+
+      def self.seen_size
+        @seen_mutex.synchronize { @seen.size }
+      end
+
+      # Forget every sighting (specs, and after a fork).
+      def self.reset_seen!
+        @seen_mutex.synchronize { @seen.clear }
+      end
+
       # Check if exception is a critical error type
       # @param exception [Exception] The exception to check
       # @return [Boolean] true if the exception is critical

data/lib/rails_error_dashboard/services/git_head_reader.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module RailsErrorDashboard
+  module Services
+    # Pure algorithm: the commit a checkout is on, read from .git directly
+    #
+    # Replaces `git rev-parse --short HEAD`. That ran in a subprocess, from the
+    # capture path, once per captured error: a fork + exec in the request
+    # thread, which SystemHealthSnapshot's safety rules forbid outright
+    # ("no fork/subprocess ever"). Three small file reads need no git binary,
+    # work in a slim container that has the .git directory but no git, and
+    # cannot hang.
+    #
+    # Resolution order, the same one git uses:
+    #   1. .git/HEAD holds a SHA (detached HEAD)
+    #   2. .git/HEAD holds "ref: refs/heads/x" -> the loose ref file
+    #   3. ...or the same ref in packed-refs
+    # `.git` may be a FILE ("gitdir: <path>") in a worktree or submodule; a
+    # worktree's refs live in the directory named by its `commondir` file.
+    #
+    # @example
+    #   GitHeadReader.call(Rails.root) # => "39f281a" or nil
+    class GitHeadReader
+      SHA = /\A\h{40}(\h{24})?\z/ # SHA-1, or SHA-256 repositories
+      SHORT = 7
+      MAX_BYTES = 1_000_000 # packed-refs in a very large repository: skip it
+
+      # @param root [Pathname, String, nil] the application root
+      # @return [String, nil] short SHA, or nil when it cannot be determined
+      def self.call(root)
+        return nil if root.nil?
+
+        git_dir = resolve_git_dir(Pathname(root.to_s).join(".git"))
+        return nil unless git_dir
+
+        head = read(git_dir.join("HEAD"))
+        return nil unless head
+        return head[0, SHORT] if head.match?(SHA)
+
+        ref = head[/\Aref:\s*(\S+)\z/, 1]
+        # A ref is a path under the git dir; never follow one out of it.
+        return nil if ref.nil? || !ref.start_with?("refs/") || ref.include?("..")
+
+        sha = [ git_dir, common_dir(git_dir) ].uniq.lazy.filter_map { |dir|
+          loose_ref(dir, ref) || packed_ref(dir, ref)
+        }.first
+        sha && sha[0, SHORT]
+      rescue => e
+        RailsErrorDashboard::Logger.debug("[RailsErrorDashboard] GitHeadReader failed: #{e.class}: #{e.message}")
+        nil
+      end
+
+      def self.resolve_git_dir(dot_git)
+        return dot_git if dot_git.directory?
+        return nil unless dot_git.file?
+
+        target = read(dot_git)&.[](/\Agitdir:\s*(.+)\z/, 1)
+        return nil unless target
+
+        dir = Pathname(target)
+        dir = dot_git.dirname.join(dir) unless dir.absolute?
+        dir.directory? ? dir : nil
+      end
+
+      def self.common_dir(git_dir)
+        relative = read(git_dir.join("commondir"))
+        return git_dir unless relative
+
+        dir = Pathname(relative)
+        dir = git_dir.join(dir) unless dir.absolute?
+        dir.directory? ? dir.cleanpath : git_dir
+      end
+
+      def self.loose_ref(dir, ref)
+        value = read(dir.join(ref))
+        value if value&.match?(SHA)
+      end
+
+      def self.packed_ref(dir, ref)
+        file = dir.join("packed-refs")
+        return nil unless file.file? && file.size <= MAX_BYTES
+
+        File.foreach(file) do |line|
+          sha, name = line.split(" ", 2)
+          return sha if name&.strip == ref && sha.match?(SHA)
+        end
+        nil
+      end
+
+      # First line of a small file, stripped; nil when absent or oversized.
+      def self.read(path)
+        return nil unless path.file? && path.size <= MAX_BYTES
+
+        File.open(path, &:gets)&.strip.presence
+      end
+
+      private_class_method :resolve_git_dir, :common_dir, :loose_ref, :packed_ref, :read
+    end
+  end
+end