solid_observer 0.4.0 → 0.6.0

data/lib/solid_observer/event_buffer_core.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+require "concurrent/timer_task"
+
+module SolidObserver
+  module EventBufferCore
+    INITIAL_METRICS = {
+      flush_failures_count: 0,
+      drops_count: 0,
+      last_flush_at: nil,
+      last_flush_duration_ms: nil,
+      last_flush_error: nil
+    }.freeze
+
+    def initialize_event_buffer
+      @mutex = Mutex.new
+      @metrics_mutex = Mutex.new
+      @buffer = []
+      @metrics = INITIAL_METRICS.dup
+      @timer_task = nil
+    end
+    private :initialize_event_buffer
+
+    def push(event_data)
+      return unless (config = SolidObserver.config).persistence_mode?
+
+      drops_count, should_flush = sync_push_and_check(event_data, config)
+      record_drop(drops_count) if drops_count.positive?
+      ensure_timer_running
+      flush! if should_flush
+    end
+
+    def flush!
+      events_to_flush = drain_buffer
+      return if events_to_flush.empty?
+
+      flush_events(events_to_flush, monotonic_ms)
+    end
+
+    def flush
+      flush!
+    end
+
+    def size
+      @mutex.synchronize { @buffer.size }
+    end
+
+    def clear
+      @mutex.synchronize { @buffer.clear }
+    end
+
+    def metrics
+      current_size = @mutex.synchronize { @buffer.size }
+      snapshot = @metrics_mutex.synchronize { @metrics.dup }
+      {
+        size: current_size,
+        max_buffer_size: SolidObserver.config.max_buffer_size
+      }.merge(snapshot)
+    end
+
+    def shutdown
+      stop_timer
+      flush!
+    end
+
+    private
+
+    def drain_buffer
+      @mutex.synchronize do
+        events_to_flush = @buffer.dup
+        @buffer.clear
+        events_to_flush
+      end
+    end
+
+    def flush_events(events_to_flush, started_at_ms)
+      flush_service.call(events_to_flush)
+      record_flush_success(monotonic_ms - started_at_ms)
+    rescue => error
+      handle_flush_error(error, events_to_flush)
+    end
+
+    def handle_flush_error(error, events_to_flush)
+      requeue_failed_events(events_to_flush)
+      record_flush_failure(error)
+      log_flush_failure(error)
+      nil
+    end
+
+    def log_flush_failure(error)
+      Rails.logger&.error("[SolidObserver] #{log_label} flush failed: #{error.message}") if defined?(Rails)
+    end
+
+    def apply_overflow_policy(event_data, config)
+      if @buffer.size < config.max_buffer_size
+        @buffer << event_data
+        return 0
+      end
+
+      handle_overflow(event_data, config.buffer_overflow_strategy)
+    end
+
+    def handle_overflow(event_data, overflow_strategy)
+      case overflow_strategy
+      when :drop_old
+        @buffer.shift
+        @buffer << event_data
+      when :drop_new
+        nil
+      else
+        raise ArgumentError, "Unsupported buffer_overflow_strategy: #{overflow_strategy.inspect}"
+      end
+
+      1
+    end
+
+    def requeue_failed_events(events_to_flush)
+      dropped_count = sync_requeue_events(events_to_flush, SolidObserver.config.max_buffer_size)
+      record_drop(dropped_count) if dropped_count.positive?
+    end
+
+    def trim_events_for_capacity(events, max_buffer_size)
+      dropped_count = events.size - max_buffer_size
+      return [events, 0] if dropped_count <= 0
+
+      [events_to_keep(events, max_buffer_size), dropped_count]
+    end
+
+    def ensure_timer_running
+      timer_to_start, timer_to_stop = replace_timer_if_stopped
+      return unless timer_to_start
+
+      timer_to_stop&.shutdown
+      timer_to_start.execute
+    end
+
+    def replace_timer_if_stopped
+      @mutex.synchronize do
+        current_timer_task = @timer_task
+        return [nil, nil] if timer_running?(current_timer_task)
+
+        [build_timer_task, current_timer_task]
+      end
+    end
+
+    def stop_timer
+      timer_to_stop = @mutex.synchronize do
+        current_timer = @timer_task
+        @timer_task = nil
+        current_timer
+      end
+
+      timer_to_stop&.shutdown
+    end
+
+    def record_flush_success(duration_ms)
+      @metrics_mutex.synchronize do
+        @metrics.merge!(
+          last_flush_at: Time.current,
+          last_flush_duration_ms: duration_ms,
+          last_flush_error: nil
+        )
+      end
+    end
+
+    def record_flush_failure(error)
+      @metrics_mutex.synchronize do
+        @metrics[:flush_failures_count] += 1
+        @metrics[:last_flush_error] = error.message
+      end
+    end
+
+    def record_drop(count = 1)
+      @metrics_mutex.synchronize do
+        @metrics[:drops_count] += count
+      end
+    end
+
+    def sync_push_and_check(event_data, config)
+      @mutex.synchronize do
+        drops_count = apply_overflow_policy(event_data, config)
+        [drops_count, @buffer.size >= config.buffer_size]
+      end
+    end
+
+    def sync_requeue_events(events_to_flush, max_buffer_size)
+      @mutex.synchronize do
+        combined_events = events_to_flush + @buffer
+        kept_events, dropped_count = trim_events_for_capacity(combined_events, max_buffer_size)
+        @buffer.replace(kept_events)
+        dropped_count
+      end
+    end
+
+    def events_to_keep(events, max_buffer_size)
+      if SolidObserver.config.buffer_overflow_strategy == :drop_old
+        events.last(max_buffer_size)
+      else
+        events.first(max_buffer_size)
+      end
+    end
+
+    def timer_running?(timer_task)
+      timer_task && !timer_task.shuttingdown?
+    end
+
+    def build_timer_task
+      @timer_task = Concurrent::TimerTask.new(
+        execution_interval: SolidObserver.config.flush_interval,
+        run_now: false
+      ) { flush! }
+    end
+
+    def monotonic_ms
+      Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+    end
+  end
+end

data/lib/solid_observer/queries/trace_query.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module SolidObserver
+  module Queries
+    class TraceQuery
+      DEFAULT_LIMIT = 100
+      CABLE_CAP = 50
+
+      Result = Struct.new(:rows, :unavailable_components, keyword_init: true)
+
+      COMPONENT_PRIORITY = {queue: 0, cache: 1, cable: 2}.freeze
+
+      COMPONENT_CONFIG = {
+        queue: {
+          enabled: :solid_queue_enabled?,
+          model: :QueueEvent,
+          columns: [:recorded_at, :event_type, :job_class, :queue_name, :duration, :id]
+        },
+        cache: {
+          enabled: :solid_cache_enabled?,
+          model: :CacheEvent,
+          columns: [:recorded_at, :event_type, :hit, :error_class, :duration, :id]
+        },
+        cable: {
+          enabled: :solid_cable_enabled?,
+          model: :CableEvent,
+          columns: [:recorded_at, :event_type, :channel_class, :broadcasting_digest, :error_class, :duration, :id]
+        }
+      }.freeze
+
+      CONNECTION_ERRORS = [
+        ActiveRecord::ConnectionNotEstablished,
+        ActiveRecord::StatementInvalid,
+        *([PG::ConnectionBad] if defined?(PG::ConnectionBad)),
+        *([Mysql2::Error::ConnectionError] if defined?(Mysql2::Error::ConnectionError)),
+        *([Trilogy::Error] if defined?(Trilogy::Error)),
+        *([SQLite3::CantOpenException] if defined?(SQLite3::CantOpenException))
+      ].freeze
+
+      def call(correlation_id:, limit: DEFAULT_LIMIT)
+        rows, unavailable = gather_rows(correlation_id)
+        effective_limit = limit.to_i
+        effective_limit = DEFAULT_LIMIT unless effective_limit.positive?
+        Result.new(rows: process_rows(rows, effective_limit), unavailable_components: unavailable)
+      end
+
+      private
+
+      def gather_rows(correlation_id)
+        COMPONENT_CONFIG.each_with_object([[], []]) do |(component, config), (rows, unavailable)|
+          next unless SolidObserver.config.public_send(config[:enabled])
+
+          component_rows = fetch_component_rows(component, config, correlation_id)
+          component_rows ? rows.concat(component_rows) : unavailable << component
+        end
+      end
+
+      def fetch_component_rows(component, config, correlation_id)
+        model = SolidObserver.const_get(config[:model])
+        return nil unless data_source_available?(model)
+
+        pluck_rows(model, component, correlation_id)
+      rescue *CONNECTION_ERRORS, TypeError
+        nil
+      end
+
+      def data_source_available?(model)
+        table_name = model.table_name.to_s
+        table_name.present? && model.connection.data_source_exists?(table_name)
+      end
+
+      def pluck_rows(model, component, correlation_id)
+        columns = COMPONENT_CONFIG[component][:columns]
+        model.where(correlation_id: correlation_id)
+          .order(:recorded_at)
+          .pluck(*columns)
+          .map { |values| build_row(component, columns, values) }
+      end
+
+      def build_row(component, columns, values)
+        row = {component: component}
+        columns.each_with_index { |column, index| row[column] = values[index] }
+        row
+      end
+
+      def process_rows(rows, limit)
+        collapsed = collapse_cable_rows(rows)
+        capped = apply_cable_cap(collapsed)
+        sorted = sort_rows(capped)
+        return sorted if capped.size <= limit
+
+        sort_rows(sorted.last(limit))
+      end
+
+      def collapse_cable_rows(rows)
+        cable_rows, other_rows = rows.partition { |row| row[:component] == :cable }
+        return rows if cable_rows.empty?
+
+        other_rows + cable_rows.chunk_while(&method(:collapse_match?)).map(&method(:collapse_group))
+      end
+
+      def collapse_match?(first_row, second_row)
+        return false unless first_row[:event_type] == "broadcast" && second_row[:event_type] == "broadcast"
+
+        digest = first_row[:broadcasting_digest]
+        digest.present? && digest == second_row[:broadcasting_digest]
+      end
+
+      def collapse_group(group)
+        row = group.first
+        size = group.size
+        row[:collapsed_count] = size if size > 1
+        row.delete(:broadcasting_digest)
+        row
+      end
+
+      def apply_cable_cap(rows)
+        cable_rows, other_rows = rows.partition { |row| row[:component] == :cable }
+        total = cable_rows.size
+        return rows if total <= CABLE_CAP
+
+        other_rows + cap_cable_rows(cable_rows, total)
+      end
+
+      def cap_cable_rows(cable_rows, total)
+        keep_count = CABLE_CAP - 1
+        [build_summary_row(cable_rows.first(total - keep_count))] + tail_rows(cable_rows, keep_count)
+      end
+
+      def tail_rows(rows, count)
+        rows.last(count)
+      end
+
+      def build_summary_row(overflow)
+        {
+          recorded_at: overflow.last[:recorded_at],
+          component: :cable,
+          event_type: "cable_summary",
+          duration: nil,
+          channel_class: nil,
+          collapsed_count: overflow.size,
+          id: -1
+        }
+      end
+
+      def sort_rows(rows)
+        rows.sort_by { |row| [row[:recorded_at], COMPONENT_PRIORITY[row[:component]], row[:id] || 0] }
+      end
+    end
+  end
+end

data/lib/solid_observer/queue_event_buffer.rb

@@ -1,221 +1,29 @@
 # frozen_string_literal: true
 
 require "singleton"
-require "concurrent/timer_task"
+
+require_relative "event_buffer_core"
 
 module SolidObserver
   # Thread-safe buffer for collecting queue events before batch insertion.
-  #
-  # Events are buffered in memory and flushed either when:
-  # - Buffer size reaches the configured threshold
-  # - Flush interval timer expires
-  #
-  # @example Push an event to the buffer
-  #   QueueEventBuffer.instance.push(event_data)
   class QueueEventBuffer
     include Singleton
+    include EventBufferCore
 
-    INITIAL_METRICS = {
-      flush_failures_count: 0,
-      drops_count: 0,
-      last_flush_at: nil,
-      last_flush_duration_ms: nil,
-      last_flush_error: nil
-    }.freeze
+    INITIAL_METRICS = EventBufferCore::INITIAL_METRICS
 
     def initialize
-      @mutex = Mutex.new
-      @metrics_mutex = Mutex.new
-      @buffer = []
-      @metrics = INITIAL_METRICS.dup
-      @timer_task = nil
-    end
-
-    # Adds an event to the buffer and triggers flush if threshold reached.
-    #
-    # @param event_data [Hash] Event data to buffer
-    # @return [void]
-    def push(event_data)
-      return unless (config = SolidObserver.config).persistence_mode?
-
-      drops_count, should_flush = sync_push_and_check(event_data, config)
-      record_drop(drops_count) if drops_count.positive?
-      ensure_timer_running
-      flush! if should_flush
-    end
-
-    # Flushes all buffered events to the database.
-    #
-    # @return [void]
-    def flush!
-      events_to_flush = nil
-
-      @mutex.synchronize do
-        return if @buffer.empty?
-        events_to_flush = @buffer.dup
-        @buffer.clear
-      end
-
-      started_at_ms = monotonic_ms
-      begin
-        Services::FlushEventBuffer.call(events_to_flush)
-      rescue => e
-        requeue_failed_events(events_to_flush)
-        record_flush_failure(e)
-        Rails.logger&.error "[SolidObserver] Buffer flush failed: #{e.message}" if defined?(Rails)
-        return
-      end
-      record_flush_success(monotonic_ms - started_at_ms)
-    end
-
-    def size
-      @mutex.synchronize { @buffer.size }
-    end
-
-    def clear
-      @mutex.synchronize { @buffer.clear }
-    end
-
-    def metrics
-      current_size = @mutex.synchronize { @buffer.size }
-      snapshot = @metrics_mutex.synchronize { @metrics.dup }
-      {
-        size: current_size,
-        max_buffer_size: SolidObserver.config.max_buffer_size
-      }.merge(snapshot)
-    end
-
-    def shutdown
-      stop_timer
-      flush!
+      initialize_event_buffer
     end
 
     private
 
-    def apply_overflow_policy(event_data, config)
-      if @buffer.size < config.max_buffer_size
-        @buffer << event_data
-        return 0
-      end
-
-      handle_overflow(event_data, config.buffer_overflow_strategy)
-    end
-
-    def handle_overflow(event_data, overflow_strategy)
-      case overflow_strategy
-      when :drop_old
-        @buffer.shift
-        @buffer << event_data
-      when :drop_new
-        # No-op: drop incoming event, keep oldest buffered events.
-      else
-        raise ArgumentError, "Unsupported buffer_overflow_strategy: #{overflow_strategy.inspect}"
-      end
-
-      1
-    end
-
-    def requeue_failed_events(events_to_flush)
-      return unless events_to_flush
-
-      dropped_count = sync_requeue_events(events_to_flush, SolidObserver.config.max_buffer_size)
-      record_drop(dropped_count) if dropped_count.positive?
-    end
-
-    def trim_events_for_capacity(events, max_buffer_size)
-      dropped_count = events.size - max_buffer_size
-      return [events, 0] if dropped_count <= 0
-
-      [events_to_keep(events, max_buffer_size), dropped_count]
-    end
-
-    def ensure_timer_running
-      timer_to_start, timer_to_stop = replace_timer_if_stopped
-      return unless timer_to_start
-
-      timer_to_stop&.shutdown
-      timer_to_start.execute
-    end
-
-    def replace_timer_if_stopped
-      @mutex.synchronize do
-        current_timer_task = @timer_task
-        return [nil, nil] if timer_running?(current_timer_task)
-
-        [build_timer_task, current_timer_task]
-      end
-    end
-
-    def stop_timer
-      timer_to_stop = @mutex.synchronize do
-        current_timer = @timer_task
-        @timer_task = nil
-        current_timer
-      end
-
-      timer_to_stop&.shutdown
-    end
-
-    def record_flush_success(duration_ms)
-      @metrics_mutex.synchronize do
-        @metrics.merge!(
-          last_flush_at: Time.current,
-          last_flush_duration_ms: duration_ms,
-          last_flush_error: nil
-        )
-      end
-    end
-
-    def record_flush_failure(error)
-      @metrics_mutex.synchronize do
-        @metrics[:flush_failures_count] += 1
-        @metrics[:last_flush_error] = error.message
-      end
-    end
-
-    def record_drop(count = 1)
-      @metrics_mutex.synchronize do
-        @metrics[:drops_count] += count
-      end
-    end
-
-    def sync_push_and_check(event_data, config)
-      @mutex.synchronize do
-        drops_count = apply_overflow_policy(event_data, config)
-        [drops_count, @buffer.size >= config.buffer_size]
-      end
-    end
-
-    def sync_requeue_events(events_to_flush, max_buffer_size)
-      @mutex.synchronize do
-        combined_events = events_to_flush + @buffer
-        kept_events, dropped_count = trim_events_for_capacity(combined_events, max_buffer_size)
-        @buffer.replace(kept_events)
-        dropped_count
-      end
-    end
-
-    def events_to_keep(events, max_buffer_size)
-      if SolidObserver.config.buffer_overflow_strategy == :drop_old
-        events.last(max_buffer_size)
-      else
-        events.first(max_buffer_size)
-      end
-    end
-
-    def timer_running?(timer_task)
-      timer_task && !timer_task.shuttingdown?
-    end
-
-    def build_timer_task
-      @timer_task = Concurrent::TimerTask.new(
-        execution_interval: SolidObserver.config.flush_interval,
-        run_now: false
-      ) { flush! }
+    def flush_service
+      Services::FlushEventBuffer
     end
 
-    def monotonic_ms
-      Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+    def log_label
+      "Buffer"
     end
   end
 end

data/lib/solid_observer/services/cable_operations.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module SolidObserver
+  module Services
+    class CableOperations
+      MESSAGES = {
+        trim: {
+          success: "Expired/trimmable Solid Cable messages trimmed.",
+          failure: "Cable trim failed. No raw Cable payloads or adapter details are shown. Use solid_observer:cable:trim if the problem continues."
+        }.freeze,
+        unavailable: "Cable controls are unavailable because Solid Cable support is disabled or not detected."
+      }.freeze
+
+      class << self
+        def available?
+          new.available?
+        end
+
+        def trim
+          new.trim
+        end
+
+        def message(operation, key = nil)
+          return MESSAGES.fetch(:unavailable) if operation == :unavailable
+
+          MESSAGES.fetch(operation).fetch(key)
+        end
+
+        def unavailable_message
+          message(:unavailable)
+        end
+      end
+
+      def available?
+        SolidObserver.config.solid_cable_enabled? && !!defined?(::SolidCable::Message)
+      end
+
+      def trim
+        messages = self.class
+        return {ok: false, message: messages.unavailable_message} unless available?
+
+        perform_operation(
+          :trim,
+          success_message: messages.message(:trim, :success),
+          failure_message: messages.message(:trim, :failure)
+        ) do
+          trim_cable_messages
+        end
+      end
+
+      private
+
+      def trim_cable_messages
+        if defined?(::SolidCable::TrimJob)
+          ::SolidCable::TrimJob.perform_now
+        else
+          ::SolidCable::Message.trimmable.delete_all
+        end
+      end
+
+      def perform_operation(name, success_message:, failure_message:)
+        yield
+        {ok: true, message: success_message}
+      rescue => error
+        log_failure(name, error)
+        {ok: false, message: failure_message}
+      end
+
+      def log_failure(name, error)
+        Rails.logger&.warn("[SolidObserver] Cable #{name} failed: #{error.class}")
+      end
+    end
+  end
+end