solid_observer 0.4.0 → 0.5.0

data/lib/solid_observer/services/cache_stats.rb

@@ -123,10 +123,10 @@ module SolidObserver
         class EventCounts
           attr_reader :error_count, :slow_count, :latest_recorded_at
 
-          def initialize
-            @error_count = 0
-            @slow_count = 0
-            @latest_recorded_at = nil
+          def initialize(error_count: 0, slow_count: 0, latest_recorded_at: nil)
+            @error_count = error_count.to_i
+            @slow_count = slow_count.to_i
+            @latest_recorded_at = latest_recorded_at
           end
 
           def record(recorded_at:, error_class:, duration:)
@@ -181,22 +181,38 @@ module SolidObserver
         attr_reader :window, :current_time
 
         def event_counts
-          counts = EventCounts.new
-
-          SolidObserver::CacheEvent.where(recorded_at: window_range).pluck(
-            :recorded_at,
-            :duration,
-            :error_class
-          ).each do |recorded_at, duration, error_class|
-            counts.record(recorded_at: recorded_at, error_class: error_class, duration: duration)
-          end
-
-          counts
+          error_count, slow_count, latest_recorded_at = SolidObserver::CacheEvent.where(recorded_at: window_range).pick(
+            Arel.sql("COUNT(CASE WHEN #{error_condition_sql} THEN 1 END)"),
+            Arel.sql("COUNT(CASE WHEN #{slow_condition_sql} THEN 1 END)"),
+            Arel.sql("MAX(CASE WHEN #{tracked_condition_sql} THEN recorded_at END)")
+          )
+
+          EventCounts.new(
+            error_count: error_count,
+            slow_count: slow_count,
+            latest_recorded_at: latest_recorded_at
+          )
         end
 
         def window_range
           (current_time - window)..current_time
         end
+
+        def tracked_condition_sql
+          "(#{error_condition_sql}) OR (#{slow_condition_sql})"
+        end
+
+        def error_condition_sql
+          "error_class IS NOT NULL AND TRIM(error_class) != ''"
+        end
+
+        def slow_condition_sql
+          "(error_class IS NULL OR TRIM(error_class) = '') AND duration >= #{slow_threshold}"
+        end
+
+        def slow_threshold
+          SolidObserver.config.cache_slow_threshold.to_f
+        end
       end
 
       class << self
@@ -218,7 +234,8 @@ module SolidObserver
         current_time = Time.current
         dashboard_response(window: window, current_time: current_time)
       rescue => error
-        error_response(error.message)
+        Rails.logger&.error("[SolidObserver] CacheStats call failed: #{error.class} #{error.message}") if defined?(Rails)
+        error_response
       end
 
       private
@@ -308,7 +325,7 @@ module SolidObserver
         numerator.to_f / denominator
       end
 
-      def error_response(message)
+      def error_response
         {
           hit_rate: 0.0,
           throughput: 0.0,
@@ -321,7 +338,7 @@ module SolidObserver
           duration_total: 0.0,
           activity_trends: ACTIVITY_TREND_EMPTY.dup,
           stability: STABILITY_EMPTY.dup,
-          error: message
+          error: "Service temporarily unavailable"
         }
       end
     end

data/lib/solid_observer/services/cleanup_storage.rb

@@ -6,6 +6,13 @@ require_relative "storage_info_snapshot"
 module SolidObserver
   module Services
     class CleanupStorage
+      MAINTENANCE_STATEMENT_BUILDERS = {
+        "sqlite" => ->(_tables) { ["VACUUM"] },
+        "postgresql" => ->(tables) { tables.map { |table_name| "VACUUM ANALYZE #{table_name}" } },
+        "mysql2" => ->(tables) { ["OPTIMIZE TABLE #{tables.join(", ")}"] },
+        "trilogy" => ->(tables) { ["OPTIMIZE TABLE #{tables.join(", ")}"] }
+      }.freeze
+
       def self.call
         new.call
       end
@@ -13,37 +20,54 @@ module SolidObserver
       def call
         return 0 if SolidObserver.config.realtime_mode?
 
-        deleted_count = perform_cleanup_transaction
-        post_cleanup(deleted_count)
-      rescue => e
-        handle_cleanup_failure(e)
+        post_cleanup(cleanup_counts)
+      rescue => error
+        handle_cleanup_failure(error)
       end
 
       private
 
+      def cleanup_counts
+        perform_cleanup.tap { record_snapshot_after_cleanup }
+      end
+
       def handle_cleanup_failure(error)
         Rails.logger.error "[SolidObserver] Cleanup failed: #{error.message}"
         raise
       end
 
-      def perform_cleanup_transaction
-        QueueEvent.transaction do
-          deleted_count = delete_old_events
-          record_snapshot_after_cleanup
-          deleted_count
-        end
-      end
-
-      def post_cleanup(deleted_count)
+      def perform_cleanup
+        config = SolidObserver.config
+        event_cutoff = config.event_retention.ago
+
+        {
+          queue_events: QueueEvent.transaction do
+            QueueEvent.where("recorded_at < ?", event_cutoff).delete_all
+          end,
+          cache_events: delete_telemetry_records(
+            SolidObserver::CacheEvent,
+            column: :recorded_at,
+            cutoff: event_cutoff
+          ),
+          cache_metrics: delete_telemetry_records(
+            SolidObserver::CacheMetric,
+            column: :period_start,
+            cutoff: config.metrics_retention.ago
+          )
+        }
+      end
+
+      def post_cleanup(cleanup_counts)
         vacuum_database
         check_storage_warnings
-        log_results(deleted_count)
-        deleted_count
+        log_results(cleanup_counts)
+        cleanup_counts.values.sum
       end
 
-      def delete_old_events
-        cutoff = SolidObserver.config.event_retention.ago
-        QueueEvent.where("recorded_at < ?", cutoff).delete_all
+      def delete_telemetry_records(model, column:, cutoff:)
+        return 0 unless data_source_available?(model)
+
+        model.where("#{column} < ?", cutoff).delete_all
       end
 
       def record_snapshot_after_cleanup
@@ -70,58 +94,69 @@ module SolidObserver
       end
 
       def vacuum_database
-        statement = maintenance_statement
-        return unless statement
-
-        QueueEvent.connection.execute(statement)
-      rescue => e
-        Rails.logger.warn "[SolidObserver] Database maintenance failed: #{e.message}"
+        maintenance_statements.each do |statement|
+          QueueEvent.connection.execute(statement)
+        end
+      rescue => error
+        Rails.logger.warn "[SolidObserver] Database maintenance failed: #{error.message}"
       end
 
       def check_storage_warnings
         current_size = current_database_size
-        return unless warning_needed?(current_size)
+        return unless current_size
+        return unless current_size > (SolidObserver.config.max_db_size * SolidObserver.config.warning_threshold)
 
         Rails.logger.warn(storage_warning_message(current_size))
       end
 
-      def warning_needed?(current_size)
-        return false unless current_size
-
-        config = SolidObserver.config
-        max_size = config.max_db_size
-        threshold = config.warning_threshold
-        current_size > (max_size * threshold)
-      end
-
       def storage_warning_message(current_size)
         max_size = SolidObserver.config.max_db_size
         percentage = ((current_size.to_f / max_size) * 100).round(1)
-        current_size_human = human_size(current_size)
-        max_size_human = human_size(max_size)
+        current_size_human = ActiveSupport::NumberHelper.number_to_human_size(
+          current_size,
+          precision: 1,
+          significant: false,
+          strip_insignificant_zeros: false
+        )
+        max_size_human = ActiveSupport::NumberHelper.number_to_human_size(
+          max_size,
+          precision: 1,
+          significant: false,
+          strip_insignificant_zeros: false
+        )
         "[SolidObserver] Queue DB approaching limit: #{current_size_human} / #{max_size_human} (#{percentage}%)"
       end
 
-      def human_size(bytes)
-        ActiveSupport::NumberHelper.number_to_human_size(bytes, precision: 1, significant: false, strip_insignificant_zeros: false)
-      end
-
       def current_database_size
         return @current_database_size if defined?(@current_database_size)
 
         @current_database_size = DatabaseSize.call(connection: QueueEvent.connection)
       end
 
-      def log_results(deleted_count)
-        Rails.logger.info "[SolidObserver] Cleaned #{deleted_count} queue events"
+      def log_results(cleanup_counts)
+        Rails.logger.info(
+          "[SolidObserver] Cleaned #{cleanup_counts[:queue_events]} queue events, " \
+          "#{cleanup_counts[:cache_events]} cache events, " \
+          "#{cleanup_counts[:cache_metrics]} cache metrics"
+        )
       end
 
-      def maintenance_statement
-        case QueueEvent.connection.adapter_name.downcase
-        when "sqlite" then "VACUUM"
-        when "postgresql" then "VACUUM ANALYZE solid_observer_queue_events"
-        when "mysql2", "trilogy" then "OPTIMIZE TABLE solid_observer_queue_events"
+      def maintenance_statements
+        tables = [QueueEvent, SolidObserver::CacheEvent, SolidObserver::CacheMetric].filter_map do |model|
+          model.table_name if data_source_available?(model)
         end
+        return [] if tables.empty?
+
+        MAINTENANCE_STATEMENT_BUILDERS.fetch(QueueEvent.connection.adapter_name.downcase, ->(_known_tables) { [] }).call(tables)
+      end
+
+      def data_source_available?(model)
+        table_name = model.table_name.to_s
+        return false if table_name.empty?
+
+        model.connection.data_source_exists?(table_name)
+      rescue *StorageInfoSnapshot::CONNECTION_ERRORS, TypeError
+        false
       end
     end
   end

data/lib/solid_observer/services/flush_cable_event_buffer.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module SolidObserver
+  module Services
+    class FlushCableEventBuffer
+      BATCH_SIZE = 100
+
+      def self.call(events)
+        new(events).call
+      end
+
+      def initialize(events)
+        @events = events
+      end
+
+      def call
+        return 0 if @events.empty?
+
+        insert_all_events
+        @events.size
+      rescue ActiveRecord::RecordInvalid, ActiveRecord::StatementInvalid => error
+        fallback_insert_count(error)
+      end
+
+      private
+
+      def insert_all_events
+        CableEvent.transaction { CableEvent.insert_all!(@events) }
+      end
+
+      def fallback_insert_count(error)
+        Rails.logger&.error("[SolidObserver] Cable bulk insert failed, retrying in batches: #{error.message}") if defined?(Rails)
+        @events.each_slice(BATCH_SIZE).sum { |batch| insert_batch(batch) }
+      end
+
+      def insert_batch(batch)
+        size = batch.size
+        CableEvent.insert_all(batch, returning: false)
+        size
+      rescue ActiveRecord::RecordInvalid, ActiveRecord::StatementInvalid => error
+        handle_batch_insert_error(size, error)
+      end
+
+      def log_batch_warning(size, error)
+        Rails.logger&.warn("[SolidObserver] Failed to insert cable batch of #{size} events: #{error.message}") if defined?(Rails)
+      end
+
+      def handle_batch_insert_error(size, error)
+        log_batch_warning(size, error)
+        0
+      end
+    end
+  end
+end

data/lib/solid_observer/services/flush_cable_metrics.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module SolidObserver
+  module Services
+    class FlushCableMetrics
+      def self.call(metrics)
+        new(metrics).call
+      end
+
+      def initialize(metrics)
+        @metrics = metrics
+      end
+
+      def call
+        return 0 if @metrics.empty?
+
+        flush_metrics
+        @metrics.size
+      rescue ActiveRecord::RecordNotUnique
+        retry
+      end
+
+      private
+
+      def flush_metrics
+        SolidObserver::CableMetric.transaction do
+          @metrics.each { |metric_data| increment_metric(metric_data) }
+        end
+      end
+
+      def increment_metric(metric_data)
+        period_start = metric_data.fetch(:period_start)
+        metric = SolidObserver::CableMetric.find_or_create_by!(period_start: period_start)
+
+        SolidObserver::CableMetric.where(id: metric.id).update_counters(update_values(metric_data))
+      end
+
+      def update_values(metric_data)
+        {
+          broadcasts_count: increment_expression(:broadcasts_count, metric_data),
+          transmissions_count: increment_expression(:transmissions_count, metric_data),
+          confirmations_count: increment_expression(:confirmations_count, metric_data),
+          rejections_count: increment_expression(:rejections_count, metric_data),
+          perform_actions_count: increment_expression(:perform_actions_count, metric_data),
+          errors_count: increment_expression(:errors_count, metric_data)
+        }
+      end
+
+      def increment_expression(column, metric_data)
+        metric_data.fetch(column, 0)
+      end
+    end
+  end
+end

data/lib/solid_observer/services/flush_cache_metrics.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module SolidObserver
+  module Services
+    class FlushCacheMetrics
+      def self.call(metrics)
+        new(metrics).call
+      end
+
+      def initialize(metrics)
+        @metrics = metrics
+      end
+
+      def call
+        return 0 if @metrics.empty?
+
+        flush_metrics
+        @metrics.size
+      rescue ActiveRecord::RecordNotUnique
+        retry
+      end
+
+      private
+
+      def flush_metrics
+        SolidObserver::CacheMetric.transaction do
+          @metrics.each { |metric_data| increment_metric(metric_data) }
+        end
+      end
+
+      def increment_metric(metric_data)
+        event_type, period_start = metric_data.values_at(:event_type, :period_start)
+        metric = SolidObserver::CacheMetric.find_or_create_by!(
+          event_type: event_type,
+          period_start: period_start
+        )
+
+        SolidObserver::CacheMetric.where(id: metric.id).update_counters(update_values(metric_data))
+      end
+
+      def update_values(metric_data)
+        {
+          operations_count: increment_expression(:operations_count, metric_data),
+          hits_count: increment_expression(:hits_count, metric_data),
+          misses_count: increment_expression(:misses_count, metric_data),
+          errors_count: increment_expression(:errors_count, metric_data),
+          duration_total: increment_expression(:duration_total, metric_data)
+        }
+      end
+
+      def increment_expression(column, metric_data)
+        metric_data.fetch(column, 0)
+      end
+    end
+  end
+end

data/lib/solid_observer/services/record_cable_event.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module SolidObserver
+  module Services
+    class RecordCableEvent
+      def self.call(event:, buffer:)
+        new(event, buffer).call
+      end
+
+      def initialize(event, buffer)
+        @event = event
+        @buffer = buffer
+      end
+
+      def call
+        record_metric_and_event
+      rescue => error
+        raise error if error.is_a?(NameError)
+        Rails.logger&.warn("[SolidObserver] Cable event recording failed: #{error.message}") if defined?(Rails)
+      end
+
+      private
+
+      def record_metric_and_event
+        SolidObserver::Services::RecordCableMetric.call(event: @event)
+        return unless should_store_event?
+
+        @buffer.push(build_event_data)
+      end
+
+      def should_store_event?
+        case @event.name
+        when "broadcast.action_cable"
+          sampled? || errored?
+        when "transmit_subscription_rejection.action_cable"
+          true
+        else
+          false
+        end
+      end
+
+      def sampled?
+        rand <= SolidObserver.config.cable_sampling_rate
+      end
+
+      def errored?
+        !exception_data.compact.empty?
+      end
+
+      def build_event_data
+        {
+          event_type: @event.name.delete_suffix(".action_cable"),
+          channel_class: channel_class,
+          broadcasting_digest: broadcasting_digest,
+          duration: duration_in_seconds,
+          error_class: exception_data[:error_class],
+          error_message: exception_data[:error_message],
+          metadata: metadata.to_json,
+          recorded_at: Time.current
+        }
+      end
+
+      def broadcasting_digest
+        return nil unless @event.name == "broadcast.action_cable"
+
+        Digest::SHA256.hexdigest(payload[:broadcasting].to_s)
+      end
+
+      def channel_class
+        class_name = payload[:channel_class]
+        return class_name if class_name
+
+        channel = payload[:channel]
+        return nil unless channel
+
+        channel.class.name || channel.to_s
+      end
+
+      def duration_in_seconds
+        @event.duration&./(1000.0)
+      end
+
+      def metadata
+        {
+          action: payload[:action]&.to_s,
+          via: payload[:via]&.to_s,
+          data_size: payload[:data]&.to_s&.bytesize,
+          message_size: payload[:message]&.to_s&.bytesize
+        }.compact
+      end
+
+      def exception_data
+        @exception_data ||= begin
+          exception_obj = payload[:exception_object]
+          exception = payload[:exception]
+
+          if exception_obj
+            {error_class: exception_obj.class.name, error_message: exception_obj.message}
+          elsif exception.is_a?(Array)
+            {error_class: exception.first, error_message: exception.last}
+          else
+            {}
+          end
+        end
+      end
+
+      def payload
+        @event.payload || {}
+      end
+    end
+  end
+end

data/lib/solid_observer/services/record_cable_metric.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require_relative "../cable_metric_buffer"
+
+module SolidObserver
+  module Services
+    class RecordCableMetric
+      COUNTERS = %i[
+        broadcasts_count
+        transmissions_count
+        confirmations_count
+        rejections_count
+        perform_actions_count
+        errors_count
+      ].freeze
+
+      EVENT_COUNTER_MAP = {
+        "broadcast.action_cable" => :broadcasts_count,
+        "transmit.action_cable" => :transmissions_count,
+        "transmit_subscription_confirmation.action_cable" => :confirmations_count,
+        "transmit_subscription_rejection.action_cable" => :rejections_count,
+        "perform_action.action_cable" => :perform_actions_count
+      }.freeze
+
+      def self.call(event:, buffer: SolidObserver::CableMetricBuffer.instance)
+        new(event, buffer).call
+      end
+
+      def initialize(event, buffer)
+        @event = event
+        @buffer = buffer
+      end
+
+      def call
+        @buffer.increment(metric_data)
+      rescue => error
+        Rails.logger&.warn("[SolidObserver] Cable metric recording failed: #{error.message}") if defined?(Rails)
+      end
+
+      private
+
+      def metric_data
+        COUNTERS.each_with_object({period_start: period_start}) do |counter, data|
+          data[counter] = (counter == target_counter) ? 1 : 0
+        end.merge(errors_count: error_increment)
+      end
+
+      def target_counter
+        EVENT_COUNTER_MAP.fetch(event_name, :broadcasts_count)
+      end
+
+      def period_start
+        Time.current.beginning_of_minute
+      end
+
+      def event_name
+        @event.name
+      end
+
+      def error_increment
+        exception_present? ? 1 : 0
+      end
+
+      def exception_present?
+        payload[:exception_object].present? || payload[:exception].is_a?(Array)
+      end
+
+      def payload
+        @event.payload || {}
+      end
+    end
+  end
+end

data/lib/solid_observer/services/record_cache_event.rb

@@ -5,6 +5,8 @@ require "digest"
 module SolidObserver
   module Services
     class RecordCacheEvent
+      INTERNAL_CACHE_KEY_PREFIX = "solid_observer/"
+
       def self.call(event:, buffer:)
         new(event, buffer).call
       end
@@ -15,6 +17,8 @@ module SolidObserver
       end
 
       def call
+        return if internal_cache_event?
+
         record_metric_and_event
       rescue => error
         raise error if error.is_a?(NameError)
@@ -137,6 +141,25 @@ module SolidObserver
       def payload
         @event.payload || {}
       end
+
+      def internal_cache_event?
+        internal_cache_key?(payload[:key])
+      end
+
+      def internal_cache_key?(key)
+        case key
+        when Array
+          nested_internal_cache_key?(key)
+        when Hash
+          nested_internal_cache_key?(key.flatten(1))
+        else
+          key.to_s.start_with?(INTERNAL_CACHE_KEY_PREFIX)
+        end
+      end
+
+      def nested_internal_cache_key?(entries)
+        entries.any? { |entry| internal_cache_key?(entry) }
+      end
     end
   end
 end