solid_observer 0.1.1 → 0.4.0

data/lib/solid_observer/services/cache_stats.rb

@@ -0,0 +1,329 @@
+# frozen_string_literal: true
+
+module SolidObserver
+  module Services
+    class CacheStats
+      RANGES = {
+        "15m" => 15.minutes,
+        "30m" => 30.minutes,
+        "1h" => 1.hour,
+        "7h" => 7.hours,
+        "1d" => 1.day,
+        "7d" => 7.days,
+        "14d" => 14.days
+      }.freeze
+      DEFAULT_RANGE = "15m"
+      ACTIVITY_TREND_EMPTY = {
+        available: false,
+        hit_rate: [],
+        operations: [],
+        errors: []
+      }.freeze
+      STABILITY_EMPTY = {
+        available: false,
+        state: :stable,
+        error_count: 0,
+        slow_count: 0,
+        latest_recorded_at: nil
+      }.freeze
+      BUCKET_RULES = [
+        [2.hours.to_i, 1.minute.to_i],
+        [1.day.to_i, 15.minutes.to_i],
+        [7.days.to_i, 2.hours.to_i]
+      ].freeze
+
+      class TrendData
+        class BucketSnapshot
+          attr_reader :operations_count, :hits_count, :misses_count, :errors_count
+
+          def initialize
+            @operations_count = 0
+            @hits_count = 0
+            @misses_count = 0
+            @errors_count = 0
+          end
+
+          def add(row)
+            @operations_count += row[1].to_i
+            @hits_count += row[2].to_i
+            @misses_count += row[3].to_i
+            @errors_count += row[4].to_i
+          end
+
+          def hit_rate
+            read_outcomes = hits_count + misses_count
+            return 0.0 if read_outcomes.zero?
+
+            hits_count.to_f / read_outcomes
+          end
+
+          def value_for(key)
+            public_send(key)
+          end
+        end
+
+        def initialize(metric_rows:, window:, current_time:)
+          @metric_rows = metric_rows
+          @window = window
+          @current_time = current_time
+        end
+
+        def to_h
+          return CacheStats::ACTIVITY_TREND_EMPTY.dup if metric_rows.empty?
+
+          buckets = blank_buckets
+          metric_rows.each do |row|
+            buckets[align_bucket(row[0].to_i)]&.add(row)
+          end
+
+          {
+            available: true,
+            hit_rate: hit_rate_series(buckets),
+            operations: count_series(buckets, :operations_count),
+            errors: count_series(buckets, :errors_count)
+          }
+        end
+
+        private
+
+        attr_reader :metric_rows, :window, :current_time
+
+        def blank_buckets
+          start_bucket = align_bucket((current_time - window).to_i)
+          end_bucket = align_bucket(current_time.to_i)
+
+          start_bucket.step(end_bucket, bucket_seconds).each_with_object({}) do |timestamp, buckets|
+            buckets[timestamp] = BucketSnapshot.new
+          end
+        end
+
+        def hit_rate_series(buckets)
+          buckets.map do |timestamp, totals|
+            {t: timestamp, v: totals.hit_rate}
+          end
+        end
+
+        def count_series(buckets, key)
+          buckets.map do |timestamp, totals|
+            {t: timestamp, v: totals.value_for(key)}
+          end
+        end
+
+        def bucket_seconds
+          seconds = window.to_i
+          CacheStats::BUCKET_RULES.find { |limit, _bucket| seconds <= limit }&.last || 4.hours.to_i
+        end
+
+        def align_bucket(value)
+          (value / bucket_seconds) * bucket_seconds
+        end
+      end
+
+      class StabilityData
+        class EventCounts
+          attr_reader :error_count, :slow_count, :latest_recorded_at
+
+          def initialize
+            @error_count = 0
+            @slow_count = 0
+            @latest_recorded_at = nil
+          end
+
+          def record(recorded_at:, error_class:, duration:)
+            kind = event_kind(error_class: error_class, duration: duration)
+            return unless kind
+
+            @latest_recorded_at = [latest_recorded_at, recorded_at].compact.max
+            @error_count += 1 if kind == :error
+            @slow_count += 1 if kind == :slow
+          end
+
+          def state
+            return :critical if error_count.positive?
+            return :degraded if slow_count.positive?
+
+            :stable
+          end
+
+          def to_h
+            {
+              available: true,
+              state: state,
+              error_count: error_count,
+              slow_count: slow_count,
+              latest_recorded_at: latest_recorded_at
+            }
+          end
+
+          private
+
+          def event_kind(error_class:, duration:)
+            return :error if error_class.present?
+            return :slow if duration.to_f >= SolidObserver.config.cache_slow_threshold.to_f
+
+            nil
+          end
+        end
+
+        def initialize(window:, current_time:)
+          @window = window
+          @current_time = current_time
+        end
+
+        def to_h
+          event_counts.to_h
+        rescue ActiveRecord::StatementInvalid
+          CacheStats::STABILITY_EMPTY.dup
+        end
+
+        private
+
+        attr_reader :window, :current_time
+
+        def event_counts
+          counts = EventCounts.new
+
+          SolidObserver::CacheEvent.where(recorded_at: window_range).pluck(
+            :recorded_at,
+            :duration,
+            :error_class
+          ).each do |recorded_at, duration, error_class|
+            counts.record(recorded_at: recorded_at, error_class: error_class, duration: duration)
+          end
+
+          counts
+        end
+
+        def window_range
+          (current_time - window)..current_time
+        end
+      end
+
+      class << self
+        def parse_range(value, fallback: DEFAULT_RANGE)
+          range_key = value.to_s
+          RANGES.key?(range_key) ? range_key : fallback
+        end
+
+        def range_duration(value, fallback: DEFAULT_RANGE)
+          RANGES.fetch(parse_range(value, fallback: fallback))
+        end
+      end
+
+      def self.call(window:)
+        new.call(window: window)
+      end
+
+      def call(window:)
+        current_time = Time.current
+        dashboard_response(window: window, current_time: current_time)
+      rescue => error
+        error_response(error.message)
+      end
+
+      private
+
+      def dashboard_response(window:, current_time:)
+        time_window = (current_time - window)..current_time
+        metric_rows = metric_rows(time_window: time_window)
+
+        build_response(
+          window: window,
+          totals: metric_totals(time_window: time_window),
+          dashboard_data: dashboard_data(window: window, current_time: current_time, metric_rows: metric_rows)
+        )
+      end
+
+      def build_response(window:, totals:, dashboard_data:)
+        operations_count, hits_count, misses_count, errors_count, duration_total = totals.values_at(
+          :operations_count,
+          :hits_count,
+          :misses_count,
+          :errors_count,
+          :duration_total
+        )
+        read_outcomes_count = hits_count + misses_count
+        window_minutes = [window.to_f / 60.0, 1.0].max
+
+        {
+          hit_rate: ratio(hits_count, read_outcomes_count),
+          throughput: operations_count.to_f / window_minutes,
+          error_rate: ratio(errors_count, operations_count),
+          avg_duration: ratio(duration_total, operations_count),
+          operations_count: operations_count,
+          hits_count: hits_count,
+          misses_count: misses_count,
+          errors_count: errors_count,
+          duration_total: duration_total,
+          activity_trends: dashboard_data[:activity_trends],
+          stability: dashboard_data[:stability]
+        }
+      end
+
+      def dashboard_data(window:, current_time:, metric_rows:)
+        {
+          activity_trends: TrendData.new(
+            metric_rows: metric_rows,
+            window: window,
+            current_time: current_time
+          ).to_h,
+          stability: StabilityData.new(window: window, current_time: current_time).to_h
+        }
+      end
+
+      def metric_rows(time_window:)
+        SolidObserver::CacheMetric.where(period_start: time_window).pluck(
+          :period_start,
+          :operations_count,
+          :hits_count,
+          :misses_count,
+          :errors_count,
+          :duration_total
+        )
+      end
+
+      def metric_totals(time_window:)
+        operations_count, hits_count, misses_count, errors_count, duration_total = SolidObserver::CacheMetric.where(
+          period_start: time_window
+        ).pick(
+          Arel.sql("COALESCE(SUM(operations_count), 0)"),
+          Arel.sql("COALESCE(SUM(hits_count), 0)"),
+          Arel.sql("COALESCE(SUM(misses_count), 0)"),
+          Arel.sql("COALESCE(SUM(errors_count), 0)"),
+          Arel.sql("COALESCE(SUM(duration_total), 0.0)")
+        )
+
+        {
+          operations_count: operations_count,
+          hits_count: hits_count,
+          misses_count: misses_count,
+          errors_count: errors_count,
+          duration_total: duration_total
+        }
+      end
+
+      def ratio(numerator, denominator)
+        return 0.0 if denominator.to_i.zero?
+
+        numerator.to_f / denominator
+      end
+
+      def error_response(message)
+        {
+          hit_rate: 0.0,
+          throughput: 0.0,
+          error_rate: 0.0,
+          avg_duration: 0.0,
+          operations_count: 0,
+          hits_count: 0,
+          misses_count: 0,
+          errors_count: 0,
+          duration_total: 0.0,
+          activity_trends: ACTIVITY_TREND_EMPTY.dup,
+          stability: STABILITY_EMPTY.dup,
+          error: message
+        }
+      end
+    end
+  end
+end

data/lib/solid_observer/services/cleanup_storage.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require_relative "database_size"
+require_relative "storage_info_snapshot"
+
 module SolidObserver
   module Services
     class CleanupStorage
@@ -10,86 +13,115 @@ module SolidObserver
       def call
         return 0 if SolidObserver.config.realtime_mode?
 
-        deleted_count = 0
+        deleted_count = perform_cleanup_transaction
+        post_cleanup(deleted_count)
+      rescue => e
+        handle_cleanup_failure(e)
+      end
+
+      private
+
+      def handle_cleanup_failure(error)
+        Rails.logger.error "[SolidObserver] Cleanup failed: #{error.message}"
+        raise
+      end
 
+      def perform_cleanup_transaction
         QueueEvent.transaction do
           deleted_count = delete_old_events
           record_snapshot_after_cleanup
+          deleted_count
         end
+      end
 
+      def post_cleanup(deleted_count)
         vacuum_database
-
         check_storage_warnings
         log_results(deleted_count)
-
         deleted_count
-      rescue => e
-        Rails.logger.error "[SolidObserver] Cleanup failed: #{e.message}"
-        raise
       end
 
-      private
-
       def delete_old_events
         cutoff = SolidObserver.config.event_retention.ago
         QueueEvent.where("recorded_at < ?", cutoff).delete_all
       end
 
       def record_snapshot_after_cleanup
-        db_size = calculate_database_size
-        event_count = QueueEvent.count
+        snapshots = StorageInfoSnapshot.call
+
+        # StorageInfo.db_size_bytes is NOT NULL; record_snapshot coerces nil to 0.
+        StorageInfo.record_snapshot(db_size: current_database_size, event_count: QueueEvent.count)
+
+        snapshots.each do |snapshot|
+          record_component_snapshot(snapshot)
+        end
+      end
+
+      def record_component_snapshot(snapshot)
+        return unless snapshot[:available]
+        component = snapshot[:component]
+        return if component == "queue_observer"
 
         StorageInfo.record_snapshot(
-          db_size: db_size,
-          event_count: event_count
+          component: component,
+          db_size: snapshot[:db_size_bytes],
+          event_count: snapshot[:event_count]
         )
       end
 
       def vacuum_database
-        adapter = QueueEvent.connection.adapter_name.downcase
-        case adapter
-        when "sqlite"
-          QueueEvent.connection.execute("VACUUM")
-        when "postgresql"
-          QueueEvent.connection.execute("VACUUM ANALYZE solid_observer_queue_events")
-        when "mysql2", "trilogy"
-          QueueEvent.connection.execute("OPTIMIZE TABLE solid_observer_queue_events")
-        end
+        statement = maintenance_statement
+        return unless statement
+
+        QueueEvent.connection.execute(statement)
       rescue => e
         Rails.logger.warn "[SolidObserver] Database maintenance failed: #{e.message}"
       end
 
-      def calculate_database_size
-        db_path = QueueEvent.connection_db_config.database
-        File.size(db_path) if File.exist?(db_path)
-      rescue => e
-        Rails.logger.warn "[SolidObserver] Could not calculate DB size: #{e.message}"
-        0
+      def check_storage_warnings
+        current_size = current_database_size
+        return unless warning_needed?(current_size)
+
+        Rails.logger.warn(storage_warning_message(current_size))
       end
 
-      def check_storage_warnings
-        max_size = SolidObserver.config.max_db_size
-        threshold = SolidObserver.config.warning_threshold
-        current_size = calculate_database_size
+      def warning_needed?(current_size)
+        return false unless current_size
 
-        return unless current_size > (max_size * threshold)
+        config = SolidObserver.config
+        max_size = config.max_db_size
+        threshold = config.warning_threshold
+        current_size > (max_size * threshold)
+      end
 
+      def storage_warning_message(current_size)
+        max_size = SolidObserver.config.max_db_size
         percentage = ((current_size.to_f / max_size) * 100).round(1)
-        Rails.logger.warn "[SolidObserver] Queue DB approaching limit: #{format_bytes(current_size)} / #{format_bytes(max_size)} (#{percentage}%)"
+        current_size_human = human_size(current_size)
+        max_size_human = human_size(max_size)
+        "[SolidObserver] Queue DB approaching limit: #{current_size_human} / #{max_size_human} (#{percentage}%)"
       end
 
-      def log_results(deleted_count)
-        Rails.logger.info "[SolidObserver] Cleaned #{deleted_count} queue events"
+      def human_size(bytes)
+        ActiveSupport::NumberHelper.number_to_human_size(bytes, precision: 1, significant: false, strip_insignificant_zeros: false)
       end
 
-      def format_bytes(bytes)
-        return "0 B" if bytes.zero?
+      def current_database_size
+        return @current_database_size if defined?(@current_database_size)
 
-        units = ["B", "KB", "MB", "GB"]
-        exp = (Math.log(bytes) / Math.log(1024)).to_i
-        exp = [exp, units.length - 1].min
+        @current_database_size = DatabaseSize.call(connection: QueueEvent.connection)
+      end
 
-        "%.1f %s" % [bytes.to_f / (1024**exp), units[exp]]
+      def log_results(deleted_count)
+        Rails.logger.info "[SolidObserver] Cleaned #{deleted_count} queue events"
+      end
+
+      def maintenance_statement
+        case QueueEvent.connection.adapter_name.downcase
+        when "sqlite" then "VACUUM"
+        when "postgresql" then "VACUUM ANALYZE solid_observer_queue_events"
+        when "mysql2", "trilogy" then "OPTIMIZE TABLE solid_observer_queue_events"
+        end
       end
     end
   end

data/lib/solid_observer/services/database_size.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module SolidObserver
+  module Services
+    # Returns bytes used by solid_observer_queue_events across supported adapters.
+    #
+    # SQLite uses whole-database page accounting; PostgreSQL and MySQL/Trilogy
+    # use table + index size from adapter-native system functions.
+    class DatabaseSize
+      DEFAULT_TABLE_NAME = "solid_observer_queue_events"
+
+      def self.call(connection:, table_name: DEFAULT_TABLE_NAME)
+        new(connection, table_name: table_name).call
+      end
+
+      def initialize(connection, table_name:)
+        @connection = connection
+        @table_name = table_name
+      end
+
+      def call
+        fetch_size
+      rescue ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished => e
+        log_query_failure(e.message)
+        nil
+      end
+
+      private
+
+      attr_reader :connection, :table_name
+
+      def adapter_key
+        case connection.adapter_name.to_s.downcase
+        when /sqlite/ then :sqlite
+        when /postgres|postgis/ then :postgresql
+        when "mysql2", "trilogy" then :mysql
+        end
+      end
+
+      def fetch_size
+        case adapter_key
+        when :sqlite then sqlite_size
+        when :postgresql then postgresql_size
+        when :mysql then mysql_size
+        else
+          unknown_adapter_size
+        end
+      end
+
+      def unknown_adapter_size
+        log_unknown_adapter
+        nil
+      end
+
+      def sqlite_size
+        page_count = connection.query_value("PRAGMA page_count")
+        page_size = connection.query_value("PRAGMA page_size")
+        return unless page_count && page_size
+
+        page_count.to_i * page_size.to_i
+      end
+
+      def postgresql_size
+        quoted_table = connection.quote(table_name)
+        connection.query_value("SELECT pg_total_relation_size(#{quoted_table})")&.to_i
+      end
+
+      def mysql_size
+        quoted_table = connection.quote(table_name)
+
+        connection.query_value(<<~SQL)&.to_i
+          SELECT COALESCE(data_length + index_length, 0)
+          FROM information_schema.tables
+          WHERE table_schema = DATABASE()
+            AND table_name = #{quoted_table}
+        SQL
+      end
+
+      def log_unknown_adapter
+        Rails.logger&.warn(
+          "[SolidObserver] Unknown adapter for DatabaseSize: " \
+          "#{connection.adapter_name.inspect} — storage monitoring disabled"
+        )
+      end
+
+      def log_query_failure(message)
+        Rails.logger&.warn("[SolidObserver] DatabaseSize query failed: #{message}")
+      end
+    end
+  end
+end

data/lib/solid_observer/services/flush_cache_event_buffer.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module SolidObserver
+  module Services
+    class FlushCacheEventBuffer
+      BATCH_SIZE = 100
+
+      def self.call(events)
+        new(events).call
+      end
+
+      def initialize(events)
+        @events = events
+      end
+
+      def call
+        return 0 if @events.empty?
+
+        insert_all_events
+        @events.size
+      rescue ActiveRecord::RecordInvalid, ActiveRecord::StatementInvalid => error
+        fallback_insert_count(error)
+      end
+
+      private
+
+      def insert_all_events
+        CacheEvent.transaction { CacheEvent.insert_all!(@events) }
+      end
+
+      def fallback_insert_count(error)
+        Rails.logger&.error("[SolidObserver] Cache bulk insert failed, retrying in batches: #{error.message}") if defined?(Rails)
+        @events.each_slice(BATCH_SIZE).sum { |batch| insert_batch(batch) }
+      end
+
+      def insert_batch(batch)
+        size = batch.size
+        CacheEvent.insert_all(batch, returning: false)
+        size
+      rescue ActiveRecord::RecordInvalid, ActiveRecord::StatementInvalid => error
+        handle_batch_insert_error(size, error)
+      end
+
+      def log_batch_warning(size, error)
+        Rails.logger&.warn("[SolidObserver] Failed to insert cache batch of #{size} events: #{error.message}") if defined?(Rails)
+      end
+
+      def handle_batch_insert_error(size, error)
+        log_batch_warning(size, error)
+        0
+      end
+    end
+  end
+end

data/lib/solid_observer/services/flush_event_buffer.rb

@@ -26,31 +26,47 @@ module SolidObserver
       def call
         return 0 if @events.empty?
 
+        bulk_insert
+        @events.size
+      rescue ActiveRecord::RecordInvalid, ActiveRecord::StatementInvalid => e
+        handle_bulk_insert_failure(e)
+      end
+
+      private
+
+      def bulk_insert
         QueueEvent.transaction do
           QueueEvent.insert_all!(@events)
         end
+      end
 
-        @events.size
-      rescue ActiveRecord::RecordInvalid, ActiveRecord::StatementInvalid => e
-        log_error("Bulk insert failed, retrying in batches: #{e.message}")
+      def handle_bulk_insert_failure(error)
+        log_error("Bulk insert failed, retrying in batches: #{error.message}")
         retry_with_smaller_batches
       end
 
-      private
-
       def retry_with_smaller_batches
-        inserted = 0
+        inserted = @events.each_slice(BATCH_SIZE).sum { |batch| insert_batch(batch) }
+        log_failed_count if @failed_count.positive?
+        inserted
+      end
 
-        @events.each_slice(BATCH_SIZE) do |batch|
-          QueueEvent.insert_all(batch, returning: false)
-          inserted += batch.size
-        rescue ActiveRecord::StatementInvalid, ActiveRecord::RecordInvalid => e
-          @failed_count += batch.size
-          log_warning("Failed to insert batch of #{batch.size} events: #{e.message}")
-        end
+      def insert_batch(batch)
+        QueueEvent.insert_all(batch, returning: false)
+        batch.size
+      rescue ActiveRecord::StatementInvalid, ActiveRecord::RecordInvalid => e
+        register_failed_batch(batch, e)
+        0
+      end
 
-        log_warning("#{@failed_count} events could not be saved") if @failed_count.positive?
-        inserted
+      def register_failed_batch(batch, error)
+        batch_size = batch.size
+        @failed_count += batch_size
+        log_warning("Failed to insert batch of #{batch_size} events: #{error.message}")
+      end
+
+      def log_failed_count
+        log_warning("#{@failed_count} events could not be saved")
       end
 
       def log_error(message)