solid_observer 0.3.0 → 0.5.0

data/lib/solid_observer/services/storage_info_snapshot.rb

@@ -0,0 +1,216 @@
+# frozen_string_literal: true
+
+require_relative "database_size"
+
+module SolidObserver
+  module Services
+    class StorageInfoSnapshot
+      class RecordCount
+        def initialize(connection, table_name)
+          @connection = connection
+          @table_name = table_name
+        end
+
+        def solid_cache_count
+          case adapter_key
+          when :postgresql then postgresql_approximate_count
+          when :mysql then mysql_approximate_count
+          else
+            yield
+          end
+        end
+
+        private
+
+        attr_reader :connection, :table_name
+
+        def adapter_key
+          case connection.adapter_name.to_s.downcase
+          when /postgres|postgis/ then :postgresql
+          when "mysql2", "trilogy", "mysql" then :mysql
+          else
+            :other
+          end
+        end
+
+        def postgresql_approximate_count
+          quoted_table = connection.quote(table_name)
+
+          connection.query_value(<<~SQL.squish)&.to_i || 0
+            SELECT COALESCE(
+              (SELECT reltuples::bigint FROM pg_class WHERE oid = to_regclass(#{quoted_table})),
+              0
+            )
+          SQL
+        end
+
+        def mysql_approximate_count
+          quoted_table = connection.quote(table_name)
+
+          connection.query_value(<<~SQL)&.to_i || 0
+            SELECT COALESCE(table_rows, 0)
+            FROM information_schema.tables
+            WHERE table_schema = DATABASE()
+              AND table_name = #{quoted_table}
+          SQL
+        end
+      end
+
+      Component = Struct.new(:key, :label, :record_label, :model, :enabled, keyword_init: true) do
+        def enabled?
+          enabled.call
+        end
+
+        def solid_cache?
+          key == "solid_cache"
+        end
+
+        def solid_cable?
+          key == "solid_cable"
+        end
+
+        def storage_model
+          model.call
+        end
+
+        def snapshot
+          return unless enabled?
+          return unavailable_snapshot(reason: "SolidCache is unavailable") if unavailable_solid_cache?
+          return unavailable_snapshot(reason: "SolidCable is unavailable") if solid_cable? && !defined?(::SolidCable::Message)
+
+          existing_data_source_snapshot
+        rescue *StorageInfoSnapshot::CONNECTION_ERRORS, TypeError
+          unavailable_snapshot(reason: "Storage unavailable")
+        end
+
+        def available_snapshot(db_size_bytes:, event_count:, **extras)
+          {
+            component: key,
+            label: label,
+            available: true,
+            db_size_bytes: db_size_bytes,
+            event_count: event_count,
+            record_label: record_label,
+            recorded_at: Time.current,
+            unavailable_reason: nil
+          }.merge(extras)
+        end
+
+        def unavailable_snapshot(reason:)
+          {
+            component: key,
+            label: label,
+            available: false,
+            db_size_bytes: nil,
+            event_count: nil,
+            record_label: record_label,
+            recorded_at: nil,
+            unavailable_reason: reason
+          }
+        end
+
+        private
+
+        def unavailable_solid_cache?
+          solid_cache? && !defined?(::SolidCache::Entry)
+        end
+
+        def existing_data_source_snapshot
+          record_model = storage_model
+          connection = record_model.connection
+          table_name = record_model.table_name.to_s
+          return unavailable_snapshot(reason: "Table unavailable") unless data_source_available?(connection, table_name)
+
+          build_available_snapshot(record_model, connection, table_name)
+        end
+
+        def build_available_snapshot(record_model, connection, table_name)
+          count = -> { record_model.count }
+          event_count = solid_cache? ? RecordCount.new(connection, table_name).solid_cache_count(&count) : count.call
+
+          available_snapshot(
+            db_size_bytes: DatabaseSize.call(connection: connection, table_name: table_name),
+            event_count: event_count,
+            **extras_for(record_model)
+          )
+        end
+
+        def extras_for(record_model)
+          solid_cable? ? solid_cable_extras(record_model) : {}
+        end
+
+        def data_source_available?(connection, table_name)
+          !table_name.empty? && connection.data_source_exists?(table_name)
+        end
+
+        def solid_cable_extras(record_model)
+          {
+            trimmable_count: safe_query { record_model.trimmable.count },
+            oldest_message_age_seconds: safe_query { (Time.current - record_model.minimum(:created_at).to_time).to_i }
+          }
+        end
+
+        def safe_query
+          yield
+        rescue *StorageInfoSnapshot::CONNECTION_ERRORS, TypeError, NoMethodError
+          nil
+        end
+      end
+
+      COMPONENTS = [
+        Component.new(
+          key: "queue_observer",
+          label: "Queue observer",
+          record_label: "observer events",
+          model: -> { SolidObserver::QueueEvent },
+          enabled: -> { SolidObserver.config.solid_queue_enabled? }
+        ),
+        Component.new(
+          key: "cache_observer",
+          label: "Cache observer",
+          record_label: "observer events",
+          model: -> { SolidObserver::CacheEvent },
+          enabled: -> { SolidObserver.config.solid_cache_enabled? }
+        ),
+        Component.new(
+          key: "solid_cache",
+          label: "SolidCache",
+          record_label: "cache rows",
+          model: -> { ::SolidCache::Entry },
+          enabled: -> { SolidObserver.config.solid_cache_enabled? }
+        ),
+        Component.new(
+          key: "cable_observer",
+          label: "Cable telemetry",
+          record_label: "observer events",
+          model: -> { SolidObserver::CableEvent },
+          enabled: -> { SolidObserver.config.solid_cable_enabled? }
+        ),
+        Component.new(
+          key: "solid_cable",
+          label: "Solid Cable messages",
+          record_label: "messages",
+          model: -> { ::SolidCable::Message },
+          enabled: -> { SolidObserver.config.solid_cable_enabled? }
+        )
+      ].freeze
+
+      CONNECTION_ERRORS = [
+        ActiveRecord::ConnectionNotEstablished,
+        ActiveRecord::StatementInvalid,
+        *([PG::ConnectionBad] if defined?(PG::ConnectionBad)),
+        *([Mysql2::Error::ConnectionError] if defined?(Mysql2::Error::ConnectionError)),
+        *([Trilogy::Error] if defined?(Trilogy::Error)),
+        *([SQLite3::CantOpenException] if defined?(SQLite3::CantOpenException))
+      ].freeze
+
+      def self.call
+        new.call
+      end
+
+      def call
+        COMPONENTS.filter_map(&:snapshot)
+      end
+    end
+  end
+end

data/lib/solid_observer/version.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module SolidObserver
-  VERSION = "0.3.0"
+  VERSION = "0.5.0"
   RUBY_MINIMUM_VERSION = "3.2.0"
   RAILS_MINIMUM_VERSION = "8.0"
 end

data/lib/solid_observer.rb

@@ -1,30 +1,55 @@
 # frozen_string_literal: true
 
+require "active_support/lazy_load_hooks"
+
 require_relative "solid_observer/version"
 require_relative "solid_observer/configuration"
 require_relative "solid_observer/correlation_id_resolver"
-require_relative "solid_observer/base_event" if defined?(ActiveRecord)
-require_relative "solid_observer/base_metric" if defined?(ActiveRecord)
 require_relative "solid_observer/params/jobs_filter"
 require_relative "solid_observer/params/events_filter"
-require_relative "solid_observer/queries/job_executions_query" if defined?(ActiveRecord)
-require_relative "solid_observer/queries/events_query" if defined?(ActiveRecord)
-require_relative "solid_observer/queries/execution_finder" if defined?(ActiveRecord)
-require_relative "solid_observer/services/record_event" if defined?(ActiveRecord)
-require_relative "solid_observer/services/flush_event_buffer" if defined?(ActiveRecord)
-require_relative "solid_observer/services/cleanup_storage" if defined?(ActiveRecord)
 require_relative "solid_observer/services/ui_auth_check"
-require_relative "solid_observer/queue_event_buffer" if defined?(ActiveRecord)
-require_relative "solid_observer/subscriber" if defined?(ActiveSupport)
+require_relative "solid_observer/subscriber"
 require_relative "solid_observer/cli/base"
 require_relative "solid_observer/cli/status"
 require_relative "solid_observer/cli/storage"
 require_relative "solid_observer/cli/jobs"
 require_relative "solid_observer/queue_stats"
-require_relative "../app/presenters/solid_observer/execution_presenter"
 require_relative "solid_observer/engine" if defined?(Rails::Engine)
 
+ActiveSupport.on_load(:active_record) do
+  require_relative "solid_observer/base_record"
+  require_relative "solid_observer/base_event"
+  require_relative "solid_observer/base_metric"
+  require_relative "solid_observer/queries/job_executions_query"
+  require_relative "solid_observer/queries/events_query"
+  require_relative "solid_observer/queries/execution_finder"
+  require_relative "solid_observer/services/record_event"
+  require_relative "solid_observer/services/flush_event_buffer"
+  require_relative "solid_observer/services/cleanup_storage"
+  require_relative "solid_observer/queue_event_buffer"
+  require_relative "solid_observer/cache_event_buffer"
+  require_relative "solid_observer/services/flush_cache_metrics"
+  require_relative "solid_observer/cache_metric_buffer"
+  require_relative "solid_observer/cache_subscriber"
+  require_relative "solid_observer/services/record_cache_event"
+  require_relative "solid_observer/services/record_cache_metric"
+  require_relative "solid_observer/services/flush_cache_event_buffer"
+  require_relative "solid_observer/services/cache_stats"
+  require_relative "solid_observer/services/cache_operations"
+  require_relative "solid_observer/cable_event_buffer"
+  require_relative "solid_observer/services/flush_cable_metrics"
+  require_relative "solid_observer/cable_metric_buffer"
+  require_relative "solid_observer/cable_subscriber"
+  require_relative "solid_observer/services/record_cable_event"
+  require_relative "solid_observer/services/record_cable_metric"
+  require_relative "solid_observer/services/flush_cable_event_buffer"
+  require_relative "solid_observer/services/cable_stats"
+  require_relative "solid_observer/services/cable_operations"
+end
+
 module SolidObserver
+  autoload :ExecutionPresenter, File.expand_path("../app/presenters/solid_observer/execution_presenter", __dir__)
+
   class Error < StandardError; end
 
   class << self

data/lib/tasks/solid_observer.rake

@@ -40,31 +40,90 @@ namespace :solid_observer do
   end
 
   namespace :buffer do
-    desc "Flush the event buffer to the database"
+    desc "Flush the event buffers to the database"
     task flush: :environment do
-      buffer = SolidObserver::QueueEventBuffer.instance
-      buffer_size = buffer.size
+      buffers = [
+        {label: "queue events", buffer: SolidObserver::QueueEventBuffer.instance},
+        {label: "cache events", buffer: SolidObserver::CacheEventBuffer.instance},
+        {label: "cache metric buckets", buffer: SolidObserver::CacheMetricBuffer.instance}
+      ]
+      buffer_sizes = buffers.to_h { |entry| [entry, entry[:buffer].size] }
+
+      if buffer_sizes.values.sum.zero?
+        puts "Buffers are empty, nothing to flush"
+      else
+        buffers.each do |entry|
+          size = buffer_sizes.fetch(entry)
+          next if size.zero?
 
-      if buffer_size.zero?
-        puts "Buffer is empty, nothing to flush"
+          puts "Flushing #{size} #{entry[:label]} from buffer..."
+          entry[:buffer].flush!
+        end
+        puts "✓ Buffers flushed successfully"
+      end
+    end
+
+    desc "Clear the event buffers without flushing to database"
+    task clear: :environment do
+      buffers = [
+        {label: "queue events", buffer: SolidObserver::QueueEventBuffer.instance},
+        {label: "cache events", buffer: SolidObserver::CacheEventBuffer.instance},
+        {label: "cache metric buckets", buffer: SolidObserver::CacheMetricBuffer.instance}
+      ]
+      buffer_sizes = buffers.to_h { |entry| [entry, entry[:buffer].size] }
+
+      if buffer_sizes.values.sum.zero?
+        puts "Buffers are already empty"
       else
-        puts "Flushing #{buffer_size} events from buffer..."
-        buffer.flush!
-        puts "✓ Buffer flushed successfully"
+        buffers.each do |entry|
+          size = buffer_sizes.fetch(entry)
+          next if size.zero?
+
+          entry[:buffer].clear
+          puts "✓ Cleared #{size} #{entry[:label]} from buffer (not saved to database)"
+        end
       end
     end
+  end
 
-    desc "Clear the event buffer without flushing to database"
+  namespace :cache do
+    desc "Clear all SolidCache entries after confirmation"
     task clear: :environment do
-      buffer = SolidObserver::QueueEventBuffer.instance
-      buffer_size = buffer.size
+      if !SolidObserver::Services::CacheOperations.available?
+        puts SolidObserver::Services::CacheOperations.unavailable_message
+        next
+      end
 
-      if buffer_size.zero?
-        puts "Buffer is already empty"
+      print "#{SolidObserver::Services::CacheOperations.message(:clear, :confirmation)} (y/N) "
+      $stdout.flush
+
+      if $stdin.gets&.strip&.downcase == "y"
+        puts SolidObserver::Services::CacheOperations.clear[:message]
       else
-        buffer.clear
-        puts "✓ Cleared #{buffer_size} events from buffer (not saved to database)"
+        puts "Aborted"
+      end
+    end
+
+    desc "Prune expired SolidCache entries"
+    task prune: :environment do
+      if !SolidObserver::Services::CacheOperations.available?
+        puts SolidObserver::Services::CacheOperations.unavailable_message
+        next
+      end
+
+      puts SolidObserver::Services::CacheOperations.prune[:message]
+    end
+  end
+
+  namespace :cable do
+    desc "Trim expired Solid Cable messages"
+    task trim: :environment do
+      if !SolidObserver::Services::CableOperations.available?
+        puts SolidObserver::Services::CableOperations.unavailable_message
+        next
       end
+
+      puts SolidObserver::Services::CableOperations.trim[:message]
     end
   end
 
@@ -75,7 +134,7 @@ namespace :solid_observer do
       puts "Running storage cleanup (retention: #{retention.inspect})..."
 
       deleted_count = SolidObserver::Services::CleanupStorage.call
-      puts "✓ Cleanup complete: #{deleted_count} old events deleted"
+      puts "✓ Cleanup complete: #{deleted_count} old telemetry rows deleted"
     end
 
     desc "Purge ALL SolidObserver storage data (use with caution!)"
@@ -85,10 +144,30 @@ namespace :solid_observer do
 
       confirmation = $stdin.gets&.strip&.downcase
       if confirmation == "y"
-        event_count = SolidObserver::QueueEvent.count
+        data_source_exists = lambda do |model|
+          table_name = model.table_name.to_s
+          !table_name.empty? && model.connection.data_source_exists?(table_name)
+        rescue ActiveRecord::ConnectionNotEstablished, ActiveRecord::StatementInvalid, TypeError
+          false
+        end
+
+        telemetry_models = [
+          {label: :queue_events, model: SolidObserver::QueueEvent},
+          {label: :cache_events, model: SolidObserver::CacheEvent},
+          {label: :cache_metrics, model: SolidObserver::CacheMetric}
+        ]
+
+        telemetry_counts = telemetry_models.to_h do |entry|
+          count = data_source_exists.call(entry[:model]) ? entry[:model].count : 0
+          [entry[:label], count]
+        end
         snapshot_count = SolidObserver::StorageInfo.count
 
-        SolidObserver::QueueEvent.delete_all
+        telemetry_models.each do |entry|
+          next unless data_source_exists.call(entry[:model])
+
+          entry[:model].delete_all
+        end
         SolidObserver::StorageInfo.delete_all
 
         connection = SolidObserver::QueueEvent.connection
@@ -97,12 +176,23 @@ namespace :solid_observer do
           connection.execute("VACUUM")
           puts "✓ Database vacuumed to reclaim disk space"
         when "postgresql"
-          connection.execute("ANALYZE solid_observer_queue_events")
-          connection.execute("ANALYZE solid_observer_storage_info")
+          [
+            SolidObserver::QueueEvent.table_name,
+            SolidObserver::StorageInfo.table_name,
+            (SolidObserver::CacheEvent.table_name if data_source_exists.call(SolidObserver::CacheEvent)),
+            (SolidObserver::CacheMetric.table_name if data_source_exists.call(SolidObserver::CacheMetric))
+          ].compact.each do |table_name|
+            connection.execute("ANALYZE #{table_name}")
+          end
           puts "✓ Database tables analyzed"
         end
 
-        puts "✓ Purged #{event_count} events and #{snapshot_count} storage snapshots"
+        puts(
+          "✓ Purged #{telemetry_counts.fetch(:queue_events)} queue events, " \
+          "#{telemetry_counts.fetch(:cache_events)} cache events, " \
+          "#{telemetry_counts.fetch(:cache_metrics)} cache metrics, " \
+          "and #{snapshot_count} storage snapshots"
+        )
       else
         puts "Aborted"
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: solid_observer
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.5.0
 platform: ruby
 authors:
 - BartOz
@@ -54,6 +54,10 @@ files:
 - app/controllers/concerns/solid_observer/require_persistence_mode.rb
 - app/controllers/concerns/solid_observer/require_solid_queue.rb
 - app/controllers/solid_observer/application_controller.rb
+- app/controllers/solid_observer/cable_dashboard_controller.rb
+- app/controllers/solid_observer/cable_operations_controller.rb
+- app/controllers/solid_observer/cache_dashboard_controller.rb
+- app/controllers/solid_observer/cache_operations_controller.rb
 - app/controllers/solid_observer/dashboard_controller.rb
 - app/controllers/solid_observer/events_controller.rb
 - app/controllers/solid_observer/jobs_controller.rb
@@ -61,11 +65,25 @@ files:
 - app/helpers/solid_observer/application_helper.rb
 - app/helpers/solid_observer/dashboard_helper.rb
 - app/jobs/solid_observer/cleanup_job.rb
+- app/models/solid_observer/cable_event.rb
+- app/models/solid_observer/cable_metric.rb
+- app/models/solid_observer/cache_event.rb
+- app/models/solid_observer/cache_metric.rb
 - app/models/solid_observer/queue_event.rb
 - app/models/solid_observer/queue_metric.rb
 - app/models/solid_observer/storage_info.rb
 - app/presenters/solid_observer/execution_presenter.rb
 - app/views/layouts/solid_observer/application.html.erb
+- app/views/solid_observer/cable_dashboard/_charts.html.erb
+- app/views/solid_observer/cable_dashboard/_recent_events.html.erb
+- app/views/solid_observer/cable_dashboard/_summary.html.erb
+- app/views/solid_observer/cable_dashboard/index.html.erb
+- app/views/solid_observer/cache_dashboard/_charts.html.erb
+- app/views/solid_observer/cache_dashboard/_recent_events.html.erb
+- app/views/solid_observer/cache_dashboard/_summary.html.erb
+- app/views/solid_observer/cache_dashboard/index.html.erb
+- app/views/solid_observer/cache_operations/_confirm_clear.html.erb
+- app/views/solid_observer/cache_operations/index.html.erb
 - app/views/solid_observer/dashboard/_chart.html.erb
 - app/views/solid_observer/dashboard/_live_state.html.erb
 - app/views/solid_observer/dashboard/_queue_table.html.erb
@@ -81,19 +99,29 @@ files:
 - app/views/solid_observer/shared/_pagination.html.erb
 - app/views/solid_observer/shared/_stat_card.html.erb
 - app/views/solid_observer/storages/show.html.erb
-- bin/console
-- bin/quality_gate
-- bin/setup
 - config/routes.rb
 - db/migrate/20260115000001_create_solid_observer_queue_events.rb
 - db/migrate/20260115000002_create_solid_observer_metrics.rb
 - db/migrate/20260115000003_create_solid_observer_storage_info.rb
 - db/migrate/20260424000001_add_composite_indexes_to_queue_events.rb
+- db/migrate/20260601000001_create_solid_observer_cache_events.rb
+- db/migrate/20260601000002_create_solid_observer_cache_metrics.rb
+- db/migrate/20260602000001_add_component_to_solid_observer_storage_infos.rb
+- db/migrate/20260612000001_add_event_type_recorded_at_index_to_cache_events.rb
+- db/migrate/20260619000001_create_solid_observer_cable_events.rb
+- db/migrate/20260619000002_create_solid_observer_cable_metrics.rb
 - lib/generators/solid_observer/install_generator.rb
 - lib/generators/solid_observer/templates/initializer.rb.tt
 - lib/solid_observer.rb
 - lib/solid_observer/base_event.rb
 - lib/solid_observer/base_metric.rb
+- lib/solid_observer/base_record.rb
+- lib/solid_observer/cable_event_buffer.rb
+- lib/solid_observer/cable_metric_buffer.rb
+- lib/solid_observer/cable_subscriber.rb
+- lib/solid_observer/cache_event_buffer.rb
+- lib/solid_observer/cache_metric_buffer.rb
+- lib/solid_observer/cache_subscriber.rb
 - lib/solid_observer/chart_buffer.rb
 - lib/solid_observer/cli/base.rb
 - lib/solid_observer/cli/jobs.rb
@@ -102,6 +130,7 @@ files:
 - lib/solid_observer/configuration.rb
 - lib/solid_observer/correlation_id_resolver.rb
 - lib/solid_observer/engine.rb
+- lib/solid_observer/event_buffer_core.rb
 - lib/solid_observer/params/events_filter.rb
 - lib/solid_observer/params/jobs_filter.rb
 - lib/solid_observer/queries/events_query.rb
@@ -109,11 +138,24 @@ files:
 - lib/solid_observer/queries/job_executions_query.rb
 - lib/solid_observer/queue_event_buffer.rb
 - lib/solid_observer/queue_stats.rb
+- lib/solid_observer/services/cable_operations.rb
+- lib/solid_observer/services/cable_stats.rb
+- lib/solid_observer/services/cache_operations.rb
+- lib/solid_observer/services/cache_stats.rb
 - lib/solid_observer/services/cleanup_storage.rb
 - lib/solid_observer/services/database_size.rb
+- lib/solid_observer/services/flush_cable_event_buffer.rb
+- lib/solid_observer/services/flush_cable_metrics.rb
+- lib/solid_observer/services/flush_cache_event_buffer.rb
+- lib/solid_observer/services/flush_cache_metrics.rb
 - lib/solid_observer/services/flush_event_buffer.rb
 - lib/solid_observer/services/install_migrations.rb
+- lib/solid_observer/services/record_cable_event.rb
+- lib/solid_observer/services/record_cable_metric.rb
+- lib/solid_observer/services/record_cache_event.rb
+- lib/solid_observer/services/record_cache_metric.rb
 - lib/solid_observer/services/record_event.rb
+- lib/solid_observer/services/storage_info_snapshot.rb
 - lib/solid_observer/services/ui_auth_check.rb
 - lib/solid_observer/subscriber.rb
 - lib/solid_observer/version.rb
@@ -141,7 +183,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Observability for the Rails 8's Solid Stack
 test_files: []

data/bin/console

@@ -1,11 +0,0 @@
-#!/usr/bin/env ruby
-# frozen_string_literal: true
-
-require "bundler/setup"
-require "solid_observer"
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-require "irb"
-IRB.start(__FILE__)