rails_error_dashboard 0.12.1 → 0.14.0

data/lib/rails_error_dashboard/queries/dashboard_stats.rb

@@ -5,6 +5,13 @@ module RailsErrorDashboard
     # Query: Fetch dashboard statistics
     # This is a read operation that aggregates error data for the dashboard
     class DashboardStats
+      # The widest window any figure on the Overview covers (total_month).
+      # The completeness predicate is matched to this, not to "today".
+      WIDEST_DISPLAYED_WINDOW = 30.days
+
+      # One instance answers ONE call: several aggregates (today's event count,
+      # the 7-day trend, spike detection) are memoised on it so that they are
+      # computed once per call rather than once per card that shows them.
       def initialize(application_id: nil)
         @application_id = application_id
       end
@@ -24,9 +31,9 @@ module RailsErrorDashboard
               # rows reported five users hitting one error as "1 error today".
               # Summing is also exact during a storm: counted-only events never
               # create occurrence rows, but they DO raise occurrence_count.
-              total_today: event_count_since(Time.current.beginning_of_day),
-              total_week: event_count_since(7.days.ago),
-              total_month: event_count_since(30.days.ago),
+              total_today: today_event_count,
+              total_week: week_event_counts.values.sum,
+              total_month: month_event_count,
               unresolved: base_scope.unresolved.count,
               resolved: base_scope.resolved.count,
               reopened: reopened_count,
@@ -51,6 +58,13 @@ module RailsErrorDashboard
               # window, the dimension is incomplete and the page says so
               # rather than presenting an undercount as fact.
               affected_users_incomplete: affected_users_incomplete?,
+              # A storm episode in this window lost its per-bucket TIMING
+              # evidence, so every time-window figure for it rests on the
+              # group's own occurred_at rather than on per-event records.
+              # Reported for the same reason as affected_users_incomplete:
+              # an incomplete dimension should say so rather than present a
+              # figure of unknown completeness as fact.
+              event_timing_incomplete: event_timing_incomplete?,
               data_unavailable: false
             }
           end
@@ -68,6 +82,7 @@ module RailsErrorDashboard
           {
             data_unavailable: true,
             affected_users_incomplete: false,
+            event_timing_incomplete: false,
             total_today: 0,
             total_week: 0,
             total_month: 0,
@@ -93,13 +108,14 @@ module RailsErrorDashboard
       end
 
       def cache_key
-        # Cache key includes last error update timestamp for auto-invalidation
-        # Also includes current hour to ensure fresh data
-        # Uses base_scope to respect application_id filter for proper cache isolation
+        # The cache GENERATION, not maximum(:updated_at): the timestamp cost a
+        # query per key build and changed on every capture, so the cache never
+        # hit while errors were arriving. Freshness after a capture is the
+        # 1-minute TTL; user actions bump the generation (AnalyticsCacheManager).
         [
           "dashboard_stats",
           @application_id || "all",
-          base_scope.maximum(:updated_at)&.to_i || 0,
+          Services::AnalyticsCacheManager.generation,
           Time.current.hour
         ].join("/")
       end
@@ -112,14 +128,29 @@ module RailsErrorDashboard
         scope
       end
 
-      # Total EVENTS in a window: the sum of every matching group's
-      # occurrence_count. platform_comparison.rb counts the same way.
+      # Total EVENTS in a window -- the events that HAPPENED in it, not the
+      # lifetime volume of the groups first seen in it.
+      #
+      # This used to filter groups by occurred_at and sum occurrence_count.
+      # occurred_at is first-seen and is never rewritten on recurrence, so an
+      # error first seen at 23:59 that recurred at 00:01 reported zero errors
+      # today and two yesterday. Queries::EventVolume counts per-event records
+      # instead: occurrence rows, plus storm-shed time buckets, plus the
+      # remainder of any group that has neither.
       def event_count_since(since)
-        base_scope.where("occurred_at >= ?", since).sum(:occurrence_count)
+        Queries::EventVolume.in_window(base_scope, since)
+      end
+
+      # The widest-window total, memoised: the stats hash shows it and the
+      # completeness predicate needs the same number. One instance answers one
+      # call, and this runs on the capture path via the stats broadcast, so a
+      # second identical aggregate here is pure waste.
+      def month_event_count
+        @month_event_count ||= event_count_since(WIDEST_DISPLAYED_WINDOW.ago)
       end
 
       def event_count_between(from, to)
-        base_scope.where("occurred_at >= ? AND occurred_at < ?", from, to).sum(:occurrence_count)
+        Queries::EventVolume.in_window(base_scope, from, to)
       end
 
       # Occurrence rows carry the user of EACH event. The group's user_id is
@@ -147,67 +178,171 @@ module RailsErrorDashboard
         recorded = occurrence_scope
                      .where("#{ErrorOccurrence.table_name}.occurred_at >= ?", Time.current.beginning_of_day)
                      .count
-        recorded < event_count_since(Time.current.beginning_of_day)
+        recorded < today_event_count
       rescue StandardError
         false
       end
 
+      # True when a storm episode overlapping this window degraded without
+      # writing its time buckets.
+      #
+      # Read from the persisted episode rather than recomputed: the flush that
+      # knew is long over by the time the dashboard renders, and the flag has
+      # to survive a replay. Every guard here is deliberate -- the storm table
+      # may not exist, and the column may not be migrated yet on an older host.
+      # True when any recorded timing gap overlaps a window this page shows.
+      #
+      # The page displays today, 7-day AND 30-day figures, so the predicate has
+      # to match the WIDEST of them. An earlier version asked only about today
+      # and dropped the warning while the weekly and monthly numbers on the
+      # same screen still contained the affected events.
+      #
+      # Read from EventTimingGap rather than the storm episode: the episode is
+      # optional (the gate can shed with its breaker closed), and it is written
+      # after the counts transaction commits. See the migration for the full
+      # history of why this moved.
+      def event_timing_incomplete?
+        # Nothing left to qualify, so no warning.
+        #
+        # Gaps are deliberately retained past their own retention while the
+        # figures they describe are still shown (see the cleanup job), so a gap
+        # can outlive every event it covered -- a group expiring is what
+        # removes those events. Warning about a window whose subject matter is
+        # gone is noise, and noise trains people to ignore the banner.
+        #
+        # Asked of surviving GROUPS, not of the event aggregate. The aggregate
+        # is the wrong witness here: losing the time buckets is exactly what
+        # makes events invisible to it, so an old group that recurs today
+        # reports zero for every window (EventVolume can only place an
+        # untracked remainder at the group's own occurred_at, which precedes
+        # the window -- see untracked_groups) while the events are real, the
+        # group is alive and a current gap records them. Reading that zero as
+        # "nothing happened" suppressed the banner in precisely the state it
+        # exists to announce.
+        #
+        # last_seen_at is the right evidence because it is what retention
+        # deletes on: a group is expired only once it has not been seen for
+        # retention_days, so "no group seen in this window" is the same fact as
+        # "the covered events are gone" -- and it is indexed, and immune to the
+        # timing loss itself.
+        #
+        # Suppressed only when BOTH witnesses are silent, because neither alone
+        # is sufficient and they fail in opposite directions. The aggregate
+        # misses an old group's untracked recurrence (the defect above); the
+        # liveness check misses the converse, since EventVolume windows
+        # occurrence rows and buckets on THEIR OWN timestamps against an
+        # unwindowed group set (see group_ids), so a group whose last_seen_at
+        # has fallen behind its own event rows can still put events on the page.
+        # Requiring both to be empty means the banner can never be dropped
+        # while any displayed figure is non-zero, which is the F22 invariant.
+        return false unless groups_seen_in_widest_window? || month_event_count.positive?
+
+        EventTimingGap.affecting?(WIDEST_DISPLAYED_WINDOW.ago, application_id: @application_id)
+      rescue StandardError
+        false
+      end
+
+      # Whether any error group was last seen inside the widest displayed
+      # window. Existence, not a count: one indexed row settles it, and this
+      # runs on the capture path via the stats broadcast.
+      #
+      # The NULL arm covers rows written before last_seen_at existed, whose
+      # occurred_at is the only timestamp they have -- the same COALESCE
+      # equivalence the retention job documents, written so each arm keeps its
+      # own index.
+      def groups_seen_in_widest_window?
+        cutoff = WIDEST_DISPLAYED_WINDOW.ago
+        base_scope.where(
+          "last_seen_at >= :cutoff OR (last_seen_at IS NULL AND occurred_at >= :cutoff)", cutoff: cutoff
+        ).exists?
+      end
+
       def reopened_count
         return 0 unless ErrorLog.column_names.include?("reopened_at")
 
         base_scope.where.not(reopened_at: nil).count
       end
 
+      # Top error types in the last 7 days, by EVENTS.
+      #
+      # Filtering groups by first-seen and summing their LIFETIME
+      # occurrence_count answered a different question: an August group that
+      # recurred today was excluded entirely, so reopening it produced an empty
+      # list while the headline total (already on EventVolume) counted the
+      # event. Routed through the same primitive as the total so the page's
+      # parts and its whole cannot drift apart.
       def top_errors
-        base_scope.where("occurred_at >= ?", 7.days.ago)
-                  .group(:error_type)
-                  .sum(:occurrence_count)
-                  .sort_by { |_, count| -count }
-                  .first(10)
-                  .to_h
+        weekly_volume.by_group_attribute(:error_type)
+                     .sort_by { |_, count| -count }
+                     .first(10)
+                     .to_h
+      end
+
+      # One EventVolume for the 7-day window, shared by every weekly breakdown.
+      def weekly_volume
+        @weekly_volume ||= Queries::EventVolume.new(base_scope, 7.days.ago)
       end
 
       # Get 7-day error trend (daily counts)
+      # Events per day, placed on the day they HAPPENED. Grouping the ErrorLog
+      # table by its own occurred_at put a group's whole lifetime volume on the
+      # day it was first seen, so a recurrence never moved the trend.
+      #
+      # The zero-filled date range is preserved: the chart needs a point for
+      # every day in the window, not only the days that had errors.
       def errors_trend_7d
-        base_scope.where("occurred_at >= ?", 7.days.ago)
-                  .group_by_day(:occurred_at, range: 7.days.ago.to_date..Date.current, default_value: 0)
-                  .sum(:occurrence_count)
+        week_event_counts
       end
 
       # Get error counts by severity for last 7 days
       # OPTIMIZED: Use database filtering instead of loading all records into Ruby
+      # Weekly events by severity.
+      #
+      # Folded from the SAME per-type breakdown top_errors uses, rather than
+      # four independently scoped sums. Two reasons: it was summing LIFETIME
+      # counts of groups born in the window (so a reopened August group scored
+      # zero across every severity), and deriving both figures from one
+      # breakdown makes them agree by construction instead of by luck --
+      # asserted by spec/queries/dashboard_breakdowns_event_volume_spec.rb.
       def errors_by_severity_7d
-        scoped_errors = base_scope.where("occurred_at >= ?", 7.days.ago)
-
-        {
-          critical: scoped_errors.where(error_type: Services::SeverityClassifier::CRITICAL_ERROR_TYPES).sum(:occurrence_count),
-          high: scoped_errors.where(error_type: Services::SeverityClassifier::HIGH_SEVERITY_ERROR_TYPES).sum(:occurrence_count),
-          medium: scoped_errors.where(error_type: Services::SeverityClassifier::MEDIUM_SEVERITY_ERROR_TYPES).sum(:occurrence_count),
-          low: scoped_errors.where.not(
-            error_type: Services::SeverityClassifier::CRITICAL_ERROR_TYPES +
-                       Services::SeverityClassifier::HIGH_SEVERITY_ERROR_TYPES +
-                       Services::SeverityClassifier::MEDIUM_SEVERITY_ERROR_TYPES
-          ).sum(:occurrence_count)
-        }
+        critical = Services::SeverityClassifier::CRITICAL_ERROR_TYPES
+        high     = Services::SeverityClassifier::HIGH_SEVERITY_ERROR_TYPES
+        medium   = Services::SeverityClassifier::MEDIUM_SEVERITY_ERROR_TYPES
+
+        totals = { critical: 0, high: 0, medium: 0, low: 0 }
+        weekly_volume.by_group_attribute(:error_type).each do |error_type, count|
+          bucket =
+            if critical.include?(error_type) then :critical
+            elsif high.include?(error_type)  then :high
+            elsif medium.include?(error_type) then :medium
+            else :low
+            end
+          totals[bucket] += count
+        end
+        totals
       end
 
       # Detect if there's an error spike
       #  Uses baselines if available, falls back to simple 2x average
+      #
+      # Memoised: the stats hash asks twice (spike_detected and spike_info), and
+      # this runs from the live stats broadcast inside the capture path.
       def spike_detected?
-        return false if errors_trend_7d.empty?
+        return @spike_detected if defined?(@spike_detected)
 
-        today_count = event_count_since(Time.current.beginning_of_day)
+        @spike_detected = compute_spike_detected
+      end
 
-        # Try baseline-based detection first
-        if baseline_anomaly_detected?(today_count)
-          return true
-        end
+      def compute_spike_detected
+        trend = errors_trend_7d
+        return false if trend.empty?
+        return true if baseline_anomalies.any?
 
         # Fall back to simple 2x average detection
-        avg_count = errors_trend_7d.values.sum / 7.0
+        avg_count = trend.values.sum / 7.0
         return false if avg_count.zero?
 
-        today_count >= (avg_count * 2)
+        today_event_count >= (avg_count * 2)
       end
 
       # Get spike information
@@ -215,7 +350,7 @@ module RailsErrorDashboard
       def spike_info
         return nil unless spike_detected?
 
-        today_count = event_count_since(Time.current.beginning_of_day)
+        today_count = today_event_count
         avg_count = (errors_trend_7d.values.sum / 7.0).round(1)
 
         info = {
@@ -226,46 +361,46 @@ module RailsErrorDashboard
         }
 
         # Add baseline info if available
-        baseline_info = baseline_anomaly_info(today_count)
+        baseline_info = baseline_anomaly_info
         info.merge!(baseline_info) if baseline_info.present?
 
         info
       end
 
-      # Check if baseline indicates anomaly
-      def baseline_anomaly_detected?(_count)
-        return false unless defined?(Queries::BaselineStats)
-
-        # Check most common error types for anomalies
-        base_scope.distinct.pluck(:error_type, :platform).compact.any? do |(error_type, platform)|
-          Queries::BaselineStats.new(error_type, platform)
-                                .check_current_anomaly(sensitivity: 2, application_id: @application_id)[:anomaly]
+      # Today, yesterday and the 7-day total all come from ONE daily breakdown.
+      # Each window is three queries (occurrence rows, shed buckets, untracked
+      # remainder), and this method runs on the capture path via the stats
+      # broadcast, so asking per window multiplied the query count.
+      def week_event_counts
+        @week_event_counts ||= begin
+          counts = Queries::EventVolume.by_day(base_scope, 7.days.ago)
+          (7.days.ago.to_date..Date.current).to_h { |day| [ day, counts[day] || 0 ] }
         end
       end
 
-      # Get baseline anomaly information
-      def baseline_anomaly_info(_total_count)
-        return nil unless defined?(Queries::BaselineStats)
+      def today_event_count
+        @today_event_count ||= week_event_counts[Date.current].to_i
+      end
 
-        # Find the most anomalous error type
-        anomalies = base_scope.distinct.pluck(:error_type, :platform).compact.map do |(error_type, platform)|
-          result = Queries::BaselineStats.new(error_type, platform)
-                                         .check_current_anomaly(sensitivity: 2, application_id: @application_id)
-          next unless result[:anomaly]
+      # Every anomalous (error_type, platform) pair, from a fixed number of
+      # queries (BaselineStats.current_anomalies), loaded once per call. This
+      # used to be about six queries per distinct pair, run twice.
+      def baseline_anomalies
+        return @baseline_anomalies if defined?(@baseline_anomalies)
 
-          {
-            error_type: error_type,
-            platform: platform,
-            count: result[:current_count],
-            level: result[:level],
-            std_devs_above: result[:std_devs_above]
-          }
-        end.compact
+        @baseline_anomalies = if defined?(Queries::BaselineStats)
+          Queries::BaselineStats.current_anomalies(sensitivity: 2, application_id: @application_id)
+        else
+          []
+        end
+      end
 
-        return nil if anomalies.empty?
+      # Get baseline anomaly information
+      def baseline_anomaly_info
+        return nil if baseline_anomalies.empty?
 
         # Return info about worst anomaly
-        worst = anomalies.max_by { |a| a[:std_devs_above] || 0 }
+        worst = baseline_anomalies.max_by { |a| a[:std_devs_above] || 0 }
         {
           baseline_detected: true,
           anomaly_error_type: worst[:error_type],
@@ -283,7 +418,7 @@ module RailsErrorDashboard
       # make a real failure percentage from, so the honest figure is the rate
       # itself, uncapped, labelled with its unit.
       def error_rate
-        today_events = event_count_since(Time.current.beginning_of_day)
+        today_events = today_event_count
         return 0.0 if today_events.zero?
 
         hours_today = ((Time.current - Time.current.beginning_of_day) / 1.hour).round(1)
@@ -299,11 +434,12 @@ module RailsErrorDashboard
       # affected user. Storm count-only events create no occurrence row, so
       # this is a floor during a storm -- affected_users_incomplete? says when.
       def affected_users_today
-        distinct_affected_users(Time.current.beginning_of_day, nil)
+        @affected_users_today ||= distinct_affected_users(Time.current.beginning_of_day, nil)
       end
 
       def affected_users_yesterday
-        distinct_affected_users(1.day.ago.beginning_of_day, Time.current.beginning_of_day)
+        @affected_users_yesterday ||=
+          distinct_affected_users(1.day.ago.beginning_of_day, Time.current.beginning_of_day)
       end
 
       def distinct_affected_users(from, to)
@@ -334,8 +470,14 @@ module RailsErrorDashboard
 
       # Calculate percentage change in errors (today vs yesterday)
       def trend_percentage
-        today = event_count_since(Time.current.beginning_of_day)
-        yesterday = event_count_between(1.day.ago.beginning_of_day, Time.current.beginning_of_day)
+        return @trend_percentage if defined?(@trend_percentage)
+
+        @trend_percentage = compute_trend_percentage
+      end
+
+      def compute_trend_percentage
+        today = today_event_count
+        yesterday = week_event_counts[Date.current - 1].to_i
 
         return 0.0 if today.zero? && yesterday.zero?
         return 100.0 if yesterday.zero? && today.positive?

data/lib/rails_error_dashboard/queries/error_correlation.rb

@@ -224,12 +224,15 @@ module RailsErrorDashboard
         previous_start = @start_date
         previous_end = current_start
 
-        current_errors = ErrorLog
+        # Both periods come from base_query, which carries the application
+        # filter (and the window start). Querying ErrorLog directly made this
+        # the one panel on an application-filtered page that counted every app.
+        current_errors = base_query
           .where("occurred_at >= ?", current_start)
           .count
 
-        previous_errors = ErrorLog
-          .where("occurred_at >= ? AND occurred_at < ?", previous_start, previous_end)
+        previous_errors = base_query
+          .where("occurred_at < ?", previous_end)
           .count
 
         change_percentage = if previous_errors > 0

data/lib/rails_error_dashboard/queries/errors_list.rb

@@ -5,6 +5,9 @@ module RailsErrorDashboard
     # Query: Fetch errors with filtering and pagination
     # This is a read operation that returns a filtered collection of errors
     class ErrorsList
+      # Escape character for LIKE patterns; see filter_by_search.
+      LIKE_ESCAPE = "!"
+
       def self.call(filters = {})
         new(filters).call
       end
@@ -129,9 +132,19 @@ module RailsErrorDashboard
         else
           # Fall back to LIKE for SQLite/MySQL - search across all relevant fields
           # Use LOWER() for case-insensitive search
-          search_pattern = "%#{@filters[:search]}%"
+          #
+          # % and _ in the term are escaped so they match themselves: a search
+          # for "snake_case" must not match "snakeXcase", and "%" must not
+          # match every row. The escape character is named explicitly because
+          # SQLite has no default one, and it is "!" rather than a backslash
+          # because a backslash inside a quoted SQL literal means different
+          # things to MySQL and to everyone else.
+          escaped = ActiveRecord::Base.sanitize_sql_like(@filters[:search].to_s, LIKE_ESCAPE)
+          search_pattern = "%#{escaped}%"
           query.where(
-            "LOWER(message) LIKE LOWER(?) OR LOWER(COALESCE(backtrace, '')) LIKE LOWER(?) OR LOWER(error_type) LIKE LOWER(?)",
+            "LOWER(message) LIKE LOWER(?) ESCAPE '#{LIKE_ESCAPE}' " \
+            "OR LOWER(COALESCE(backtrace, '')) LIKE LOWER(?) ESCAPE '#{LIKE_ESCAPE}' " \
+            "OR LOWER(error_type) LIKE LOWER(?) ESCAPE '#{LIKE_ESCAPE}'",
             search_pattern, search_pattern, search_pattern
           )
         end