rails_error_dashboard 0.8.1 → 0.8.3

data/lib/rails_error_dashboard/queries/rack_attack_summary.rb

@@ -2,9 +2,15 @@
 
 module RailsErrorDashboard
   module Queries
-    # Query: Aggregate Rack Attack events from breadcrumbs across all errors
-    # Scans error_logs breadcrumbs JSON, filters for "rack_attack" category crumbs,
-    # and groups by rule name with counts, unique IPs, paths, and error associations.
+    # Query: Aggregate Rack Attack events from the rack_attack_events table.
+    #
+    # Previously this scanned every error_log's breadcrumbs JSON in Ruby, which was
+    # both expensive (full-table scan + JSON.parse per row) and incomplete (events
+    # were only stored when an unrelated error happened to occur in the same
+    # request). Events now have their own table, so this is indexed SQL.
+    #
+    # Returns rows grouped by rule with counts, unique discriminators (IPs),
+    # the most frequent path, and last-seen timestamps.
     class RackAttackSummary
       def self.call(days = 30, application_id: nil)
         new(days, application_id: application_id).call
@@ -25,65 +31,70 @@ module RailsErrorDashboard
       private
 
       def base_query
-        scope = ErrorLog.where("occurred_at >= ?", @start_date)
-                        .where.not(breadcrumbs: nil)
-        scope = scope.where(application_id: @application_id) if @application_id.present?
+        scope = RackAttackEvent.where("period_hour >= ?", @start_date)
+        scope = scope.for_application(@application_id) if @application_id.present?
         scope
       end
 
       def aggregated_events
-        results = {}
+        rows = base_query.pluck(
+          :rule, :match_type, :discriminator, :path, :event_count, :last_seen_at, :period_hour
+        )
+        return [] if rows.empty?
 
-        base_query.select(:id, :breadcrumbs, :occurred_at).find_each(batch_size: 500) do |error_log|
-          crumbs = parse_breadcrumbs(error_log.breadcrumbs)
-          next if crumbs.empty?
+        grouped = {}
 
-          rack_attack_crumbs = crumbs.select { |c| c["c"] == "rack_attack" }
-          next if rack_attack_crumbs.empty?
+        rows.each do |rule, match_type, discriminator, path, event_count, last_seen_at, period_hour|
+          key = rule.to_s.presence || "unknown"
+          count = event_count.to_i
+          seen_at = last_seen_at || period_hour
 
-          rack_attack_crumbs.each do |crumb|
-            meta = crumb["meta"] || {}
-            rule = meta["rule"].to_s.presence || "unknown"
+          entry = grouped[key] ||= {
+            rule: key,
+            match_type: match_type.to_s,
+            count: 0,
+            ips: Set.new,
+            path_counts: Hash.new(0),
+            last_seen: nil
+          }
 
-            if results[rule]
-              results[rule][:count] += 1
-              results[rule][:ips] << meta["discriminator"].to_s if meta["discriminator"].present?
-              results[rule][:paths] << meta["path"].to_s if meta["path"].present?
-              results[rule][:error_ids] << error_log.id
-              results[rule][:last_seen] = [ results[rule][:last_seen], error_log.occurred_at ].compact.max
-            else
-              results[rule] = {
-                rule: rule,
-                match_type: meta["type"].to_s,
-                count: 1,
-                ips: Set.new([ meta["discriminator"].to_s ].reject(&:blank?)),
-                paths: Set.new([ meta["path"].to_s ].reject(&:blank?)),
-                error_ids: [ error_log.id ],
-                last_seen: error_log.occurred_at
-              }
-            end
-          end
+          entry[:count] += count
+          entry[:ips] << discriminator.to_s if discriminator.present?
+          entry[:path_counts][path.to_s] += count if path.present?
+          entry[:last_seen] = [ entry[:last_seen], seen_at ].compact.max
+
+          # Prefer the most severe match type when a rule spans several. A rule
+          # that both tracks and blocks should surface as "blocklist".
+          entry[:match_type] = match_type.to_s if severity(match_type) > severity(entry[:match_type])
         end
 
-        results.values.each do |r|
-          r[:error_ids] = r[:error_ids].uniq
-          r[:error_count] = r[:error_ids].size
+        grouped.values.each do |r|
+          # "Top path" now means genuinely most-frequent, not first-seen.
+          r[:top_path] = r[:path_counts].max_by { |_path, count| count }&.first
+          r[:paths] = r[:path_counts].sort_by { |_p, c| -c }.map(&:first)
           r[:unique_ips] = r[:ips].size
-          r[:top_path] = r[:paths].first
           r[:ips] = r[:ips].to_a
-          r[:paths] = r[:paths].to_a
+          # Distinct rate-limited clients is the meaningful figure here; the old
+          # breadcrumb-derived :error_count no longer applies now that events are
+          # stored independently of errors.
+          r[:error_count] = 0
+          r.delete(:path_counts)
         end
-        results.values.sort_by { |r| -r[:count] }
+
+        grouped.values.sort_by { |r| -r[:count] }
       rescue => e
         Rails.logger.error("[RailsErrorDashboard] RackAttackSummary query failed: #{e.class}: #{e.message}")
         []
       end
 
-      def parse_breadcrumbs(raw)
-        return [] if raw.blank?
-        JSON.parse(raw)
-      rescue JSON::ParserError
-        []
+      # Ordering used to pick the most severe match type for a rule.
+      def severity(match_type)
+        case match_type.to_s
+        when "blocklist" then 3
+        when "throttle"  then 2
+        when "track"     then 1
+        else 0
+        end
       end
     end
   end

data/lib/rails_error_dashboard/queries/storm_history.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Queries
+    # Query: Storm protection episode history + active-storm lookup.
+    #
+    # Read-only. Powers the /errors/storms page and the layout banner.
+    class StormHistory
+      RECENT_BANNER_WINDOW = 24.hours
+
+      def self.call(limit: 50)
+        return { active: nil, recent: nil, events: [] } unless StormEvent.table_exists?
+
+        {
+          active: StormEvent.active.recent_first.first,
+          recent: StormEvent.ended_within(RECENT_BANNER_WINDOW).recent_first.first,
+          events: StormEvent.recent_first.limit(limit).to_a
+        }
+      rescue => e
+        RailsErrorDashboard::Logger.error(
+          "[RailsErrorDashboard] StormHistory query failed: #{e.message}"
+        )
+        { active: nil, recent: nil, events: [] }
+      end
+
+      # Cheap banner lookup for the layout — one indexed query on the happy
+      # path (no active storm), two when a banner is showing.
+      def self.banner_event
+        return nil unless RailsErrorDashboard.configuration.enable_storm_protection
+        return nil unless StormEvent.table_exists?
+
+        StormEvent.active.recent_first.first ||
+          StormEvent.ended_within(RECENT_BANNER_WINDOW).recent_first.first
+      rescue
+        nil
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/services/rack_attack_tracker.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Services
+    # Buffers Rack::Attack events in a thread-local hash and flushes them to the
+    # database asynchronously.
+    #
+    # WHY THIS EXISTS (issue #143): Rack::Attack events were previously only
+    # recorded as breadcrumbs. Breadcrumbs are harvested exclusively by LogError,
+    # so an event was only ever persisted if an unrelated exception happened to be
+    # raised later in the same request. A throttled request returns HTTP 429 and
+    # raises nothing, so the event was always discarded when ErrorCatcher cleared
+    # the buffer. This tracker persists events independently of error capture.
+    #
+    # Events are aggregated by (rule, match_type, discriminator, path, method) and
+    # counted, rather than stored one row per event — a rate-limit flood is exactly
+    # when we must not do one INSERT per request.
+    #
+    # SAFETY RULES (HOST_APP_SAFETY.md):
+    # - Zero I/O in the record path (hash lookup + integer increment)
+    # - Never raises — every public method wrapped in rescue
+    # - Thread-local state, no mutex needed
+    # - LRU eviction bounds memory (rotating-IP attacks cannot grow it unbounded)
+    # - Async flush via background job
+    class RackAttackTracker
+      COUNTS_THREAD_KEY = :red_rack_attack_counts
+      FLUSH_THREAD_KEY  = :red_rack_attack_last_flush
+
+      # Field length caps — must match the column limits in the migration so that
+      # truncation happens before the value ever reaches the unique upsert index.
+      MAX_RULE_LENGTH          = 250
+      MAX_DISCRIMINATOR_LENGTH = 191
+      MAX_PATH_LENGTH          = 191
+      MAX_METHOD_LENGTH        = 10
+
+      # Separator for the composite buffer key. Chosen because it cannot appear in
+      # an HTTP method and is vanishingly unlikely in a rule name or path.
+      KEY_SEPARATOR = "
"
+
+      class << self
+        # Record a single Rack::Attack event. Called from the AS::Notifications
+        # subscriber on every throttle/blocklist/track match.
+        #
+        # @param rule [String] matched rule name (env["rack.attack.matched"])
+        # @param match_type [String] "throttle" | "blocklist" | "track"
+        # @param discriminator [String] rate-limit key (usually IP or user id)
+        # @param path [String] request path
+        # @param http_method [String] request method
+        def record(rule:, match_type:, discriminator: nil, path: nil, http_method: nil)
+          return unless enabled?
+
+          key = build_key(
+            truncate(rule, MAX_RULE_LENGTH),
+            match_type.to_s,
+            truncate(discriminator, MAX_DISCRIMINATOR_LENGTH),
+            truncate(path, MAX_PATH_LENGTH),
+            truncate(http_method, MAX_METHOD_LENGTH)
+          )
+
+          counts = (Thread.current[COUNTS_THREAD_KEY] ||= {})
+          counts[key] = (counts[key] || 0) + 1
+
+          # LRU eviction — Ruby hashes preserve insertion order, so the first key
+          # is the oldest. Bounds memory under rotating-discriminator attacks.
+          evict_oldest!(counts) if counts.size > max_cache_size
+
+          maybe_flush!
+          nil
+        rescue => e
+          RailsErrorDashboard::Logger.debug(
+            "[RailsErrorDashboard] RackAttackTracker.record failed: #{e.class} - #{e.message}"
+          )
+          nil
+        end
+
+        # Flush buffered counts to the database (async by default).
+        # Clears the thread-local buffer before dispatching so a slow/failed
+        # flush cannot double-count on the next call.
+        def flush!(sync: false)
+          counts = Thread.current[COUNTS_THREAD_KEY]
+          return if counts.nil? || counts.empty?
+
+          snapshot = counts.dup
+          counts.clear
+          Thread.current[FLUSH_THREAD_KEY] = Time.now.to_f
+
+          dispatch_flush(snapshot, sync: sync)
+          nil
+        rescue => e
+          RailsErrorDashboard::Logger.debug(
+            "[RailsErrorDashboard] RackAttackTracker.flush! failed: #{e.class} - #{e.message}"
+          )
+          nil
+        end
+
+        # Clear thread-local state without persisting. Used by specs and by
+        # thread teardown paths.
+        def reset!
+          Thread.current[COUNTS_THREAD_KEY] = nil
+          Thread.current[FLUSH_THREAD_KEY] = nil
+          nil
+        rescue => e
+          nil
+        end
+
+        # Current buffered counts (inspection / specs). Non-destructive.
+        def buffered_counts
+          (Thread.current[COUNTS_THREAD_KEY] || {}).dup
+        rescue => e
+          {}
+        end
+
+        # Decompose a buffer key back into its parts.
+        # @return [Array<String>] [rule, match_type, discriminator, path, http_method]
+        def parse_key(key)
+          key.to_s.split(KEY_SEPARATOR, 5)
+        end
+
+        private
+
+        def enabled?
+          RailsErrorDashboard.configuration.enable_rack_attack_tracking
+        rescue => e
+          false
+        end
+
+        def build_key(*parts)
+          parts.map(&:to_s).join(KEY_SEPARATOR)
+        end
+
+        def evict_oldest!(hash)
+          oldest_key = hash.each_key.first
+          hash.delete(oldest_key) if oldest_key
+        end
+
+        # Cheap periodic flush check — a float subtraction, no I/O.
+        def maybe_flush!
+          now = Time.now.to_f
+          last_flush = Thread.current[FLUSH_THREAD_KEY] ||= now
+          return unless (now - last_flush) >= flush_interval
+
+          flush!
+        end
+
+        # Dispatch asynchronously so the request path never waits on the DB.
+        # Falls back to a synchronous write if enqueueing fails (e.g. no queue
+        # backend configured) — losing the events would be worse, and this only
+        # happens on the flush interval, not per event.
+        def dispatch_flush(snapshot, sync: false)
+          return if snapshot.empty?
+
+          if sync
+            Commands::FlushRackAttackEvents.call(counts: snapshot)
+          else
+            RackAttackFlushJob.perform_later(snapshot)
+          end
+        rescue => e
+          RailsErrorDashboard::Logger.debug(
+            "[RailsErrorDashboard] RackAttackTracker.dispatch_flush enqueue failed, " \
+            "falling back to sync: #{e.class} - #{e.message}"
+          )
+          begin
+            Commands::FlushRackAttackEvents.call(counts: snapshot)
+          rescue => inner
+            RailsErrorDashboard::Logger.debug(
+              "[RailsErrorDashboard] RackAttackTracker sync fallback failed: #{inner.class} - #{inner.message}"
+            )
+          end
+        end
+
+        def max_cache_size
+          RailsErrorDashboard.configuration.rack_attack_max_cache_size || 1000
+        rescue => e
+          1000
+        end
+
+        def flush_interval
+          RailsErrorDashboard.configuration.rack_attack_flush_interval || 60
+        rescue => e
+          60
+        end
+
+        # Truncate to the column limit and strip the key separator. A rule name
+        # or path containing KEY_SEPARATOR would otherwise shift every field on
+        # parse — silently writing the discriminator into the path column.
+        def truncate(str, max)
+          s = str.to_s
+          s = s.delete(KEY_SEPARATOR) if s.include?(KEY_SEPARATOR)
+          s.length > max ? s[0, max] : s
+        end
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/services/storm_protection/circuit_breaker.rb

@@ -0,0 +1,195 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Services
+    module StormProtection
+      # Per-process circuit breaker for the error capture path.
+      #
+      # Counts capture attempts in fixed 10-second buckets and transitions
+      # between states based on the completed bucket's rate:
+      #
+      #   :closed    — normal operation, per-fingerprint buckets decide fidelity
+      #   :shedding  — elevated rate: context shed, notifications suppressed
+      #   :open      — storm: count-only mode, zero per-event I/O
+      #   :half_open — post-cooldown probe: small sample admitted, watching rate
+      #
+      # Hysteresis: opens FAST (a single hot bucket, or mid-bucket fast-trip),
+      # closes SLOW (two consecutive calm buckets) to prevent flapping.
+      #
+      # Concurrency: the hot path is one AtomicFixnum increment plus a float
+      # comparison. The mutex is taken only on bucket roll (once per 10s) and
+      # for state transitions — never per event.
+      class CircuitBreaker
+        BUCKET_SECONDS = 10
+        CALM_BUCKETS_TO_CLOSE = 2
+
+        attr_reader :state
+
+        # @param clock [#call] returns monotonic seconds; injectable for tests
+        def initialize(clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
+          @clock = clock
+          @mutex = Mutex.new
+          reset!
+        end
+
+        def reset!
+          @mutex.synchronize do
+            @state = :closed
+            @bucket_start = @clock.call
+            @bucket_count = Concurrent::AtomicFixnum.new(0)
+            @calm_buckets = 0
+            @opened_at = nil
+            @episode = nil
+          end
+        end
+
+        # Count one capture attempt and return the state that should govern it.
+        # Called on EVERY capture — must stay allocation-free on the fast path.
+        def record!
+          now = @clock.call
+          roll!(now) if now - @bucket_start >= BUCKET_SECONDS
+
+          count = @bucket_count.increment
+
+          # Fast-trip: don't wait for the bucket to complete if it's already
+          # over the open threshold — at 50k errors/min a full 10s bucket
+          # would let ~8k events through before reacting.
+          if count >= open_threshold * BUCKET_SECONDS && @state != :open
+            trip_open!(now, count)
+          end
+
+          @state
+        end
+
+        # Episode metadata for the honesty layer (storm_events row).
+        # @return [Hash, nil] nil when no episode is active or recently closed
+        def episode_snapshot
+          @mutex.synchronize { @episode&.dup }
+        end
+
+        # Forget a closed episode once it has been persisted by the flush job.
+        def clear_closed_episode!
+          @mutex.synchronize do
+            @episode = nil if @episode && @episode[:ended_at]
+          end
+        end
+
+        private
+
+        def roll!(now)
+          @mutex.synchronize do
+            elapsed = now - @bucket_start
+            return if elapsed < BUCKET_SECONDS # another thread already rolled
+
+            rate = @bucket_count.value / elapsed.to_f
+            @bucket_start = now
+            @bucket_count = Concurrent::AtomicFixnum.new(0)
+            transition!(rate, now)
+          end
+        end
+
+        # Transition table — runs inside @mutex, once per bucket roll.
+        # track_peak runs AFTER the case: a transition out of :closed creates
+        # the episode, and the triggering bucket's rate must be its first peak.
+        def transition!(rate, now)
+          case @state
+          when :closed
+            if rate >= open_threshold
+              open!(now)
+            elsif rate >= shedding_threshold
+              enter!(:shedding, now)
+            end
+          when :shedding
+            if rate >= open_threshold
+              open!(now)
+            elsif rate < shedding_threshold / 2.0
+              calm_step!(now)
+            else
+              @calm_buckets = 0
+            end
+          when :open
+            if now - @opened_at >= cooldown_seconds && rate < shedding_threshold
+              @state = :half_open
+              @calm_buckets = 0
+            end
+          when :half_open
+            if rate >= shedding_threshold
+              open!(now)
+            else
+              calm_step!(now)
+            end
+          end
+
+          track_peak(rate)
+        end
+
+        def calm_step!(now)
+          @calm_buckets += 1
+          close!(now) if @calm_buckets >= CALM_BUCKETS_TO_CLOSE
+        end
+
+        def enter!(new_state, _now)
+          begin_episode! if @state == :closed
+          @state = new_state
+          @calm_buckets = 0
+        end
+
+        def open!(now)
+          begin_episode! if @state == :closed
+          @state = :open
+          @opened_at = now
+          @calm_buckets = 0
+          @episode[:reached_open] = true if @episode
+        end
+
+        # Mid-bucket fast trip — takes the mutex (rare: at most once per storm onset).
+        def trip_open!(now, count)
+          @mutex.synchronize do
+            return if @state == :open
+
+            begin_episode! if @state == :closed
+            track_peak(count / [ now - @bucket_start, 1.0 ].max)
+            @state = :open
+            @opened_at = now
+            @calm_buckets = 0
+            @episode[:reached_open] = true if @episode
+          end
+        end
+
+        def close!(_now)
+          @state = :closed
+          @calm_buckets = 0
+          @episode[:ended_at] = Time.current if @episode
+        end
+
+        def begin_episode!
+          @episode = {
+            started_at: Time.current,
+            ended_at: nil,
+            peak_rate_per_minute: 0,
+            reached_open: false
+          }
+        end
+
+        def track_peak(rate_per_second)
+          return unless @episode
+
+          per_minute = (rate_per_second * 60).round
+          @episode[:peak_rate_per_minute] = per_minute if per_minute > @episode[:peak_rate_per_minute]
+        end
+
+        def shedding_threshold
+          RailsErrorDashboard.configuration.storm_shedding_threshold_per_second.to_f
+        end
+
+        def open_threshold
+          RailsErrorDashboard.configuration.storm_open_threshold_per_second.to_f
+        end
+
+        def cooldown_seconds
+          RailsErrorDashboard.configuration.storm_cooldown_seconds.to_i
+        end
+      end
+    end
+  end
+end

data/lib/rails_error_dashboard/services/storm_protection/count_buffer.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module RailsErrorDashboard
+  module Services
+    module StormProtection
+      # In-memory accumulator for events that are counted but not stored
+      # per-event (Layer 1 overflow and the breaker's count-only mode).
+      #
+      # Stores exact counts plus just enough identity to reconcile onto the
+      # right ErrorLog at flush time: the flush command recomputes the
+      # canonical error_hash from these parts (with application resolved in
+      # the background job, where DB access is allowed) and issues a single
+      # `occurrence_count = occurrence_count + N` UPDATE per fingerprint.
+      # Fingerprints first seen during count-only mode get a minimal ErrorLog
+      # created from the stored exemplar. Counting is exact — no extrapolation.
+      #
+      # Memory: bounded map; beyond the cap events land in a single overflow
+      # counter (still exact in total, anonymous in identity).
+      #
+      # Concurrency: snapshot! atomically swaps the whole map out via
+      # AtomicReference, so flushing never races with recording.
+      class CountBuffer
+        Entry = Struct.new(
+          :error_class, :message, :first_app_frame,
+          :controller_name, :action_name, :custom_hash,
+          :count, :first_seen_at, :last_seen_at
+        )
+
+        def initialize
+          reset!
+        end
+
+        def reset!
+          @map_ref = Concurrent::AtomicReference.new(Concurrent::Map.new)
+          @overflow = Concurrent::AtomicFixnum.new(0)
+        end
+
+        # Record one counted-not-stored event.
+        # @param gate_key [String] cheap in-process bucketing key
+        # @param parts [Hash] identity parts captured at the gate
+        def record(gate_key, parts)
+          map = @map_ref.get
+          entry = map[gate_key]
+
+          unless entry
+            if map.size >= max_tracked
+              @overflow.increment
+              return
+            end
+            entry = map.compute_if_absent(gate_key) do
+              Entry.new(
+                parts[:error_class], parts[:message], parts[:first_app_frame],
+                parts[:controller_name], parts[:action_name], parts[:custom_hash],
+                Concurrent::AtomicFixnum.new(0), Time.current, Time.current
+              )
+            end
+          end
+
+          entry.count.increment
+          entry.last_seen_at = Time.current
+        end
+
+        def any?
+          @overflow.value.positive? || !@map_ref.get.empty?
+        end
+
+        # Atomically swap the buffer out and return serializable entry hashes.
+        # @return [Hash] { entries: Array<Hash>, overflow: Integer }
+        def snapshot!
+          old_map = @map_ref.get_and_set(Concurrent::Map.new)
+          overflow = @overflow.value
+          @overflow.update { |v| v - overflow }
+
+          entries = []
+          old_map.each_pair do |_key, entry|
+            entries << {
+              "error_class" => entry.error_class,
+              "message" => entry.message,
+              "first_app_frame" => entry.first_app_frame,
+              "controller_name" => entry.controller_name,
+              "action_name" => entry.action_name,
+              "custom_hash" => entry.custom_hash,
+              "count" => entry.count.value,
+              "first_seen_at" => entry.first_seen_at.iso8601,
+              "last_seen_at" => entry.last_seen_at.iso8601
+            }
+          end
+
+          { entries: entries, overflow: overflow }
+        end
+
+        private
+
+        def max_tracked
+          RailsErrorDashboard.configuration.storm_max_tracked_fingerprints.to_i
+        end
+      end
+    end
+  end
+end