solid_observer 0.4.0 → 0.6.0

data/lib/solid_observer/services/cable_stats.rb

@@ -0,0 +1,385 @@
+# frozen_string_literal: true
+
+require_relative "storage_info_snapshot"
+
+module SolidObserver
+  module Services
+    # :reek:TooManyConstants
+    class CableStats
+      RANGES = {
+        "15m" => 15.minutes,
+        "30m" => 30.minutes,
+        "1h" => 1.hour,
+        "7h" => 7.hours,
+        "1d" => 1.day,
+        "7d" => 7.days,
+        "14d" => 14.days
+      }.freeze
+      DEFAULT_RANGE = "15m"
+      ACTIVITY_TREND_EMPTY = {
+        available: false,
+        broadcasts: [],
+        rejections: []
+      }.freeze
+      STABILITY_EMPTY = {
+        available: false,
+        state: :stable,
+        rejection_count: 0,
+        error_count: 0,
+        rejection_rate: 0.0,
+        backlog_ratio: nil,
+        backlog_available: false,
+        latest_recorded_at: nil
+      }.freeze
+      STABILITY_DEGRADED = {
+        available: true,
+        state: :degraded,
+        rejection_count: 0,
+        error_count: 0,
+        rejection_rate: 0.0,
+        backlog_ratio: nil,
+        backlog_available: false,
+        latest_recorded_at: nil
+      }.freeze
+      BUCKET_RULES = [
+        [2.hours.to_i, 1.minute.to_i],
+        [1.day.to_i, 15.minutes.to_i],
+        [7.days.to_i, 2.hours.to_i]
+      ].freeze
+
+      class TrendData
+        class BucketSnapshot
+          attr_reader :broadcasts_count, :rejections_count
+
+          def initialize
+            @broadcasts_count = 0
+            @rejections_count = 0
+          end
+
+          def add(row)
+            @broadcasts_count += row[1].to_i
+            @rejections_count += row[4].to_i
+          end
+
+          def value_for(key)
+            public_send(key)
+          end
+        end
+
+        def initialize(metric_rows:, window:, current_time:)
+          @metric_rows = metric_rows
+          @window = window
+          @current_time = current_time
+        end
+
+        def to_h
+          return CableStats::ACTIVITY_TREND_EMPTY.dup if metric_rows.empty?
+
+          buckets = blank_buckets
+          metric_rows.each do |row|
+            buckets[align_bucket(row[0].to_i)]&.add(row)
+          end
+
+          {
+            available: true,
+            broadcasts: count_series(buckets, :broadcasts_count),
+            rejections: count_series(buckets, :rejections_count)
+          }
+        end
+
+        private
+
+        attr_reader :metric_rows, :window, :current_time
+
+        def blank_buckets
+          start_bucket = align_bucket((current_time - window).to_i)
+          end_bucket = align_bucket(current_time.to_i)
+
+          start_bucket.step(end_bucket, bucket_seconds).each_with_object({}) do |timestamp, buckets|
+            buckets[timestamp] = BucketSnapshot.new
+          end
+        end
+
+        def count_series(buckets, key)
+          buckets.map do |timestamp, totals|
+            {t: timestamp, v: totals.value_for(key)}
+          end
+        end
+
+        def bucket_seconds
+          seconds = window.to_i
+          CableStats::BUCKET_RULES.find { |limit, _bucket| seconds <= limit }&.last || 4.hours.to_i
+        end
+
+        def align_bucket(value)
+          (value / bucket_seconds) * bucket_seconds
+        end
+      end
+
+      class BacklogSnapshot
+        def self.call
+          new.call
+        end
+
+        # :reek:TooManyStatements
+        def call
+          snapshot = fetch_snapshot
+          return unavailable unless snapshot && snapshot[:available]
+
+          count = snapshot[:trimmable_count].to_i
+          total = snapshot[:event_count].to_i
+
+          {
+            available: true,
+            count: count,
+            ratio: total.positive? ? count.to_f / total.to_f : 0.0
+          }
+        rescue *StorageInfoSnapshot::CONNECTION_ERRORS, TypeError, NoMethodError
+          unavailable
+        end
+
+        private
+
+        def fetch_snapshot
+          StorageInfoSnapshot.call.find { |component| component[:component] == "solid_cable" }
+        end
+
+        def unavailable
+          {available: false, count: nil, ratio: nil}
+        end
+      end
+
+      class StabilityData
+        def initialize(window:, current_time:, backlog_snapshot:)
+          @window = window
+          @current_time = current_time
+          @backlog_snapshot = backlog_snapshot
+        end
+
+        def to_h(metric_broadcasts_count:, metric_rejections_count:)
+          compute(
+            metric_broadcasts_count: metric_broadcasts_count,
+            metric_rejections_count: metric_rejections_count
+          ).to_h
+        rescue ActiveRecord::StatementInvalid
+          CableStats::STABILITY_DEGRADED.dup
+        end
+
+        private
+
+        attr_reader :window, :current_time, :backlog_snapshot
+
+        # :reek:TooManyStatements
+        def compute(metric_broadcasts_count:, metric_rejections_count:)
+          event_counts = query_event_counts
+          rejection_count = event_counts[:rejection_count]
+          error_count = event_counts[:error_count]
+          rejection_rate = CableStats.ratio(metric_rejections_count, metric_broadcasts_count)
+          rejection_present = rejection_count.to_i.positive? || metric_rejections_count.to_i.positive?
+          backlog_ratio = backlog_snapshot[:ratio]
+          backlog_available = backlog_snapshot[:available]
+
+          {
+            available: true,
+            state: stability_state(
+              error_count: error_count,
+              rejection_rate: rejection_rate,
+              rejection_present: rejection_present,
+              backlog_ratio: backlog_ratio,
+              backlog_available: backlog_available
+            ),
+            rejection_count: rejection_count,
+            error_count: error_count,
+            rejection_rate: rejection_rate,
+            backlog_ratio: backlog_ratio,
+            backlog_available: backlog_available,
+            latest_recorded_at: event_counts[:latest_recorded_at]
+          }
+        end
+
+        # :reek:ControlParameter
+        # :reek:LongParameterList
+        # :reek:TooManyStatements
+        def stability_state(error_count:, rejection_rate:, rejection_present:, backlog_ratio:, backlog_available:)
+          config = SolidObserver.config
+
+          return :critical if error_count.to_i > config.cable_error_threshold.to_i
+          return :critical if rejection_rate >= config.cable_rejection_threshold.to_f
+          return :critical if backlog_ratio && backlog_ratio >= 0.5
+          return :degraded if backlog_ratio && backlog_ratio >= config.cable_backlog_threshold.to_f
+          return :degraded if rejection_present
+          return :degraded unless backlog_available
+
+          :stable
+        end
+
+        def query_event_counts
+          rejection_count, error_count, latest_recorded_at = SolidObserver::CableEvent.where(recorded_at: window_range).pick(
+            Arel.sql("COUNT(CASE WHEN event_type = 'transmit_subscription_rejection' THEN 1 END)"),
+            Arel.sql("COUNT(CASE WHEN error_class IS NOT NULL AND TRIM(error_class) != '' THEN 1 END)"),
+            Arel.sql("MAX(CASE WHEN event_type = 'transmit_subscription_rejection' OR (error_class IS NOT NULL AND TRIM(error_class) != '') THEN recorded_at END)")
+          )
+
+          {
+            rejection_count: rejection_count.to_i,
+            error_count: error_count.to_i,
+            latest_recorded_at: parse_latest_recorded_at(latest_recorded_at)
+          }
+        end
+
+        def parse_latest_recorded_at(value)
+          return nil unless value.present?
+
+          value.is_a?(Time) ? value : Time.parse(value.to_s)
+        rescue ArgumentError
+          nil
+        end
+
+        def window_range
+          (current_time - window)..current_time
+        end
+      end
+
+      class << self
+        def parse_range(value, fallback: DEFAULT_RANGE)
+          range_key = value.to_s
+          RANGES.key?(range_key) ? range_key : fallback
+        end
+
+        def range_duration(value, fallback: DEFAULT_RANGE)
+          RANGES.fetch(parse_range(value, fallback: fallback))
+        end
+
+        def ratio(numerator, denominator)
+          return 0.0 if denominator.to_i.zero?
+
+          numerator.to_f / denominator.to_f
+        end
+      end
+
+      def self.call(window:)
+        new.call(window: window)
+      end
+
+      def call(window:)
+        current_time = Time.current
+        dashboard_response(window: window, current_time: current_time)
+      rescue => error
+        Rails.logger&.error("[SolidObserver] CableStats call failed: #{error.class} #{error.message}") if defined?(Rails)
+        error_response
+      end
+
+      private
+
+      def dashboard_response(window:, current_time:)
+        time_window = (current_time - window)..current_time
+        metric_rows = metric_rows(time_window: time_window)
+        totals = metric_totals(time_window: time_window)
+        backlog_snapshot = BacklogSnapshot.call
+
+        build_response(
+          totals: totals,
+          backlog_snapshot: backlog_snapshot,
+          dashboard_data: dashboard_data(
+            window: window,
+            current_time: current_time,
+            metric_rows: metric_rows,
+            totals: totals,
+            backlog_snapshot: backlog_snapshot
+          )
+        )
+      end
+
+      # :reek:FeatureEnvy
+      # :reek:LongParameterList
+      def build_response(totals:, backlog_snapshot:, dashboard_data:)
+        broadcasts_count = totals[:broadcasts_count]
+        rejections_count = totals[:rejections_count]
+
+        {
+          broadcasts_count: broadcasts_count,
+          transmissions_count: totals[:transmissions_count],
+          confirmations_count: totals[:confirmations_count],
+          rejections_count: rejections_count,
+          perform_actions_count: totals[:perform_actions_count],
+          errors_count: totals[:errors_count],
+          rejection_rate: self.class.ratio(rejections_count, broadcasts_count),
+          activity_trends: dashboard_data[:activity_trends],
+          stability: dashboard_data[:stability],
+          backlog_count: backlog_snapshot[:count],
+          backlog_available: backlog_snapshot[:available]
+        }
+      end
+
+      # :reek:LongParameterList
+      def dashboard_data(window:, current_time:, metric_rows:, totals:, backlog_snapshot:)
+        {
+          activity_trends: TrendData.new(
+            metric_rows: metric_rows,
+            window: window,
+            current_time: current_time
+          ).to_h,
+          stability: StabilityData.new(
+            window: window,
+            current_time: current_time,
+            backlog_snapshot: backlog_snapshot
+          ).to_h(
+            metric_broadcasts_count: totals[:broadcasts_count],
+            metric_rejections_count: totals[:rejections_count]
+          )
+        }
+      end
+
+      def metric_rows(time_window:)
+        SolidObserver::CableMetric.where(period_start: time_window).pluck(
+          :period_start,
+          :broadcasts_count,
+          :transmissions_count,
+          :confirmations_count,
+          :rejections_count,
+          :perform_actions_count,
+          :errors_count
+        )
+      end
+
+      def metric_totals(time_window:)
+        broadcasts_count, transmissions_count, confirmations_count, rejections_count, perform_actions_count, errors_count = SolidObserver::CableMetric.where(
+          period_start: time_window
+        ).pick(
+          Arel.sql("COALESCE(SUM(broadcasts_count), 0)"),
+          Arel.sql("COALESCE(SUM(transmissions_count), 0)"),
+          Arel.sql("COALESCE(SUM(confirmations_count), 0)"),
+          Arel.sql("COALESCE(SUM(rejections_count), 0)"),
+          Arel.sql("COALESCE(SUM(perform_actions_count), 0)"),
+          Arel.sql("COALESCE(SUM(errors_count), 0)")
+        )
+
+        {
+          broadcasts_count: broadcasts_count,
+          transmissions_count: transmissions_count,
+          confirmations_count: confirmations_count,
+          rejections_count: rejections_count,
+          perform_actions_count: perform_actions_count,
+          errors_count: errors_count
+        }
+      end
+
+      def error_response
+        {
+          broadcasts_count: 0,
+          transmissions_count: 0,
+          confirmations_count: 0,
+          rejections_count: 0,
+          perform_actions_count: 0,
+          errors_count: 0,
+          rejection_rate: 0.0,
+          activity_trends: ACTIVITY_TREND_EMPTY.dup,
+          stability: STABILITY_EMPTY.dup,
+          backlog_count: nil,
+          backlog_available: false,
+          error: "Service temporarily unavailable"
+        }
+      end
+    end
+  end
+end

data/lib/solid_observer/services/cache_stats.rb

@@ -123,10 +123,10 @@ module SolidObserver
         class EventCounts
           attr_reader :error_count, :slow_count, :latest_recorded_at
 
-          def initialize
-            @error_count = 0
-            @slow_count = 0
-            @latest_recorded_at = nil
+          def initialize(error_count: 0, slow_count: 0, latest_recorded_at: nil)
+            @error_count = error_count.to_i
+            @slow_count = slow_count.to_i
+            @latest_recorded_at = latest_recorded_at
           end
 
           def record(recorded_at:, error_class:, duration:)
@@ -181,22 +181,38 @@ module SolidObserver
         attr_reader :window, :current_time
 
         def event_counts
-          counts = EventCounts.new
-
-          SolidObserver::CacheEvent.where(recorded_at: window_range).pluck(
-            :recorded_at,
-            :duration,
-            :error_class
-          ).each do |recorded_at, duration, error_class|
-            counts.record(recorded_at: recorded_at, error_class: error_class, duration: duration)
-          end
-
-          counts
+          error_count, slow_count, latest_recorded_at = SolidObserver::CacheEvent.where(recorded_at: window_range).pick(
+            Arel.sql("COUNT(CASE WHEN #{error_condition_sql} THEN 1 END)"),
+            Arel.sql("COUNT(CASE WHEN #{slow_condition_sql} THEN 1 END)"),
+            Arel.sql("MAX(CASE WHEN #{tracked_condition_sql} THEN recorded_at END)")
+          )
+
+          EventCounts.new(
+            error_count: error_count,
+            slow_count: slow_count,
+            latest_recorded_at: latest_recorded_at
+          )
         end
 
         def window_range
           (current_time - window)..current_time
         end
+
+        def tracked_condition_sql
+          "(#{error_condition_sql}) OR (#{slow_condition_sql})"
+        end
+
+        def error_condition_sql
+          "error_class IS NOT NULL AND TRIM(error_class) != ''"
+        end
+
+        def slow_condition_sql
+          "(error_class IS NULL OR TRIM(error_class) = '') AND duration >= #{slow_threshold}"
+        end
+
+        def slow_threshold
+          SolidObserver.config.cache_slow_threshold.to_f
+        end
       end
 
       class << self
@@ -218,7 +234,8 @@ module SolidObserver
         current_time = Time.current
         dashboard_response(window: window, current_time: current_time)
       rescue => error
-        error_response(error.message)
+        Rails.logger&.error("[SolidObserver] CacheStats call failed: #{error.class} #{error.message}") if defined?(Rails)
+        error_response
       end
 
       private
@@ -308,7 +325,7 @@ module SolidObserver
         numerator.to_f / denominator
       end
 
-      def error_response(message)
+      def error_response
         {
           hit_rate: 0.0,
           throughput: 0.0,
@@ -321,7 +338,7 @@ module SolidObserver
           duration_total: 0.0,
           activity_trends: ACTIVITY_TREND_EMPTY.dup,
           stability: STABILITY_EMPTY.dup,
-          error: message
+          error: "Service temporarily unavailable"
         }
       end
     end

data/lib/solid_observer/services/cleanup_storage.rb

@@ -6,6 +6,13 @@ require_relative "storage_info_snapshot"
 module SolidObserver
   module Services
     class CleanupStorage
+      MAINTENANCE_STATEMENT_BUILDERS = {
+        "sqlite" => ->(_tables) { ["VACUUM"] },
+        "postgresql" => ->(tables) { tables.map { |table_name| "VACUUM ANALYZE #{table_name}" } },
+        "mysql2" => ->(tables) { ["OPTIMIZE TABLE #{tables.join(", ")}"] },
+        "trilogy" => ->(tables) { ["OPTIMIZE TABLE #{tables.join(", ")}"] }
+      }.freeze
+
       def self.call
         new.call
       end
@@ -13,37 +20,54 @@ module SolidObserver
       def call
         return 0 if SolidObserver.config.realtime_mode?
 
-        deleted_count = perform_cleanup_transaction
-        post_cleanup(deleted_count)
-      rescue => e
-        handle_cleanup_failure(e)
+        post_cleanup(cleanup_counts)
+      rescue => error
+        handle_cleanup_failure(error)
       end
 
       private
 
+      def cleanup_counts
+        perform_cleanup.tap { record_snapshot_after_cleanup }
+      end
+
       def handle_cleanup_failure(error)
         Rails.logger.error "[SolidObserver] Cleanup failed: #{error.message}"
         raise
       end
 
-      def perform_cleanup_transaction
-        QueueEvent.transaction do
-          deleted_count = delete_old_events
-          record_snapshot_after_cleanup
-          deleted_count
-        end
-      end
-
-      def post_cleanup(deleted_count)
+      def perform_cleanup
+        config = SolidObserver.config
+        event_cutoff = config.event_retention.ago
+
+        {
+          queue_events: QueueEvent.transaction do
+            QueueEvent.where("recorded_at < ?", event_cutoff).delete_all
+          end,
+          cache_events: delete_telemetry_records(
+            SolidObserver::CacheEvent,
+            column: :recorded_at,
+            cutoff: event_cutoff
+          ),
+          cache_metrics: delete_telemetry_records(
+            SolidObserver::CacheMetric,
+            column: :period_start,
+            cutoff: config.metrics_retention.ago
+          )
+        }
+      end
+
+      def post_cleanup(cleanup_counts)
         vacuum_database
         check_storage_warnings
-        log_results(deleted_count)
-        deleted_count
+        log_results(cleanup_counts)
+        cleanup_counts.values.sum
       end
 
-      def delete_old_events
-        cutoff = SolidObserver.config.event_retention.ago
-        QueueEvent.where("recorded_at < ?", cutoff).delete_all
+      def delete_telemetry_records(model, column:, cutoff:)
+        return 0 unless data_source_available?(model)
+
+        model.where("#{column} < ?", cutoff).delete_all
       end
 
       def record_snapshot_after_cleanup
@@ -70,58 +94,69 @@ module SolidObserver
       end
 
       def vacuum_database
-        statement = maintenance_statement
-        return unless statement
-
-        QueueEvent.connection.execute(statement)
-      rescue => e
-        Rails.logger.warn "[SolidObserver] Database maintenance failed: #{e.message}"
+        maintenance_statements.each do |statement|
+          QueueEvent.connection.execute(statement)
+        end
+      rescue => error
+        Rails.logger.warn "[SolidObserver] Database maintenance failed: #{error.message}"
       end
 
       def check_storage_warnings
         current_size = current_database_size
-        return unless warning_needed?(current_size)
+        return unless current_size
+        return unless current_size > (SolidObserver.config.max_db_size * SolidObserver.config.warning_threshold)
 
         Rails.logger.warn(storage_warning_message(current_size))
       end
 
-      def warning_needed?(current_size)
-        return false unless current_size
-
-        config = SolidObserver.config
-        max_size = config.max_db_size
-        threshold = config.warning_threshold
-        current_size > (max_size * threshold)
-      end
-
       def storage_warning_message(current_size)
         max_size = SolidObserver.config.max_db_size
         percentage = ((current_size.to_f / max_size) * 100).round(1)
-        current_size_human = human_size(current_size)
-        max_size_human = human_size(max_size)
+        current_size_human = ActiveSupport::NumberHelper.number_to_human_size(
+          current_size,
+          precision: 1,
+          significant: false,
+          strip_insignificant_zeros: false
+        )
+        max_size_human = ActiveSupport::NumberHelper.number_to_human_size(
+          max_size,
+          precision: 1,
+          significant: false,
+          strip_insignificant_zeros: false
+        )
         "[SolidObserver] Queue DB approaching limit: #{current_size_human} / #{max_size_human} (#{percentage}%)"
       end
 
-      def human_size(bytes)
-        ActiveSupport::NumberHelper.number_to_human_size(bytes, precision: 1, significant: false, strip_insignificant_zeros: false)
-      end
-
       def current_database_size
         return @current_database_size if defined?(@current_database_size)
 
         @current_database_size = DatabaseSize.call(connection: QueueEvent.connection)
       end
 
-      def log_results(deleted_count)
-        Rails.logger.info "[SolidObserver] Cleaned #{deleted_count} queue events"
+      def log_results(cleanup_counts)
+        Rails.logger.info(
+          "[SolidObserver] Cleaned #{cleanup_counts[:queue_events]} queue events, " \
+          "#{cleanup_counts[:cache_events]} cache events, " \
+          "#{cleanup_counts[:cache_metrics]} cache metrics"
+        )
       end
 
-      def maintenance_statement
-        case QueueEvent.connection.adapter_name.downcase
-        when "sqlite" then "VACUUM"
-        when "postgresql" then "VACUUM ANALYZE solid_observer_queue_events"
-        when "mysql2", "trilogy" then "OPTIMIZE TABLE solid_observer_queue_events"
+      def maintenance_statements
+        tables = [QueueEvent, SolidObserver::CacheEvent, SolidObserver::CacheMetric].filter_map do |model|
+          model.table_name if data_source_available?(model)
         end
+        return [] if tables.empty?
+
+        MAINTENANCE_STATEMENT_BUILDERS.fetch(QueueEvent.connection.adapter_name.downcase, ->(_known_tables) { [] }).call(tables)
+      end
+
+      def data_source_available?(model)
+        table_name = model.table_name.to_s
+        return false if table_name.empty?
+
+        model.connection.data_source_exists?(table_name)
+      rescue *StorageInfoSnapshot::CONNECTION_ERRORS, TypeError
+        false
       end
     end
   end