dead_bro 0.2.2 → 0.2.3

data/lib/dead_bro/collectors/jobs.rb

@@ -0,0 +1,403 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "time"
+
+module DeadBro
+  module Collectors
+    # Jobs collector is responsible for building the unified "jobs" payload
+    # that the background job queue monitor sends once per interval.
+    #
+    # It encapsulates queue backend detection (Sidekiq, SolidQueue, etc.)
+    # and enriches the payload with optional database, process, and system
+    # statistics.
+    module Jobs
+      module_function
+
+      # Public entry point: returns a single Hash suitable for JSON encoding.
+      def collect
+        queue_system = detect_queue_system
+
+        payload = {
+          queue_system: queue_system
+        }
+
+        # Queue backend–specific stats
+        payload[:queue] = collect_sidekiq_stats if queue_system == :sidekiq
+        payload[:queue] = collect_solid_queue_stats if queue_system == :solid_queue
+        payload[:queue] = collect_delayed_job_stats if queue_system == :delayed_job
+        payload[:queue] = collect_good_job_stats if queue_system == :good_job
+
+        # Optional collectors moved to Monitor
+
+        payload
+      rescue => e
+        {
+          error_class: e.class.name,
+          error_message: e.message.to_s[0, 500]
+        }
+      end
+
+      # --- Queue system detection -------------------------------------------------
+
+      def detect_queue_system
+        return :sidekiq if defined?(Sidekiq)
+        return :solid_queue if defined?(SolidQueue)
+        return :delayed_job if defined?(Delayed::Job)
+        return :good_job if defined?(GoodJob)
+        :unknown
+      end
+
+      # --- Sidekiq collector ------------------------------------------------------
+
+      def collect_sidekiq_stats
+        return {} unless defined?(Sidekiq)
+
+        stats = {
+          processed: nil,
+          failed: nil,
+          enqueued: nil,
+          scheduled_size: nil,
+          retry_size: nil,
+          dead_size: nil,
+          workers_size: nil,
+          processes_size: nil,
+          memory_rss_bytes: nil,
+          queues: []
+        }
+
+        begin
+          sidekiq_stats = safe_sidekiq_stats
+          if sidekiq_stats
+            stats[:processed] = begin
+              sidekiq_stats.processed
+            rescue
+              nil
+            end
+            stats[:failed] = begin
+              sidekiq_stats.failed
+            rescue
+              nil
+            end
+            stats[:enqueued] = begin
+              sidekiq_stats.enqueued
+            rescue
+              nil
+            end
+            stats[:scheduled_size] = begin
+              sidekiq_stats.scheduled_size
+            rescue
+              nil
+            end
+            stats[:retry_size] = begin
+              sidekiq_stats.retry_size
+            rescue
+              nil
+            end
+            stats[:dead_size] = begin
+              sidekiq_stats.dead_size
+            rescue
+              nil
+            end
+            stats[:workers_size] = begin
+              sidekiq_stats.workers_size
+            rescue
+              nil
+            end
+            stats[:processes_size] = begin
+              sidekiq_stats.processes_size
+            rescue
+              nil
+            end
+          end
+
+          # Per-queue size and latency
+          queue_class = begin
+            Sidekiq.const_get(:Queue)
+          rescue
+            nil
+          end
+          if queue_class && queue_class.respond_to?(:all)
+            queue_class.all.each do |queue|
+              stats[:queues] << {
+                name: queue.name,
+                size: safe_integer(queue.size),
+                latency_s: safe_latency(queue)
+              }
+            end
+          end
+
+          # Process RSS at collection time (best-effort)
+          if DeadBro.configuration.respond_to?(:enable_process_stats) && DeadBro.configuration.enable_process_stats
+            stats[:memory_rss_bytes] = begin
+              ProcessInfo.rss_bytes
+            rescue
+              nil
+            end
+          end
+        rescue => e
+          stats[:error_class] = e.class.name
+          stats[:error_message] = e.message.to_s[0, 500]
+        end
+
+        stats
+      rescue => e
+        {
+          error_class: e.class.name,
+          error_message: e.message.to_s[0, 500]
+        }
+      end
+
+      def safe_sidekiq_stats
+        require "sidekiq/api"
+        Sidekiq::Stats.new
+      rescue LoadError, NameError, StandardError
+        nil
+      end
+
+      def safe_integer(value)
+        Integer(value)
+      rescue
+        nil
+      end
+
+      def safe_latency(queue)
+        latency = begin
+          queue.latency
+        rescue
+          nil
+        end
+        return nil unless latency
+
+        value = begin
+          Float(latency)
+        rescue
+          nil
+        end
+        return nil unless value && value.finite?
+
+        value
+      rescue
+        nil
+      end
+
+      # --- SolidQueue collector (database-backed) ---------------------------------
+
+      def collect_solid_queue_stats
+        return {} unless defined?(SolidQueue)
+        return {} unless defined?(ActiveRecord)
+        return {} unless ActiveRecord::Base.respond_to?(:connected?) && ActiveRecord::Base.connected?
+
+        stats = {total_queued: 0, total_busy: 0, queues: {}}
+
+        begin
+          conn = ActiveRecord::Base.connection
+          return stats unless conn.respond_to?(:table_exists?) && conn.table_exists?("solid_queue_jobs")
+
+          # queued jobs
+          result = conn.execute("SELECT queue_name, COUNT(*) as count FROM solid_queue_jobs WHERE finished_at IS NULL GROUP BY queue_name")
+          parse_query_result(result).each do |row|
+            queue_name = (row["queue_name"] || row[:queue_name] || "default").to_s
+            count = (row["count"] || row[:count] || 0).to_i
+            stats[:queues][queue_name] = {queued: count, busy: 0, scheduled: 0, retries: 0}
+            stats[:total_queued] += count
+          end
+
+          # busy jobs
+          result = conn.execute("SELECT queue_name, COUNT(*) as count FROM solid_queue_jobs WHERE finished_at IS NULL AND claimed_at IS NOT NULL GROUP BY queue_name")
+          parse_query_result(result).each do |row|
+            queue_name = (row["queue_name"] || row[:queue_name] || "default").to_s
+            count = (row["count"] || row[:count] || 0).to_i
+            stats[:queues][queue_name] ||= {queued: 0, busy: 0, scheduled: 0, retries: 0}
+            stats[:queues][queue_name][:busy] = count
+            stats[:total_busy] += count
+          end
+
+          # scheduled jobs
+          result = conn.execute("SELECT COUNT(*) as count FROM solid_queue_jobs WHERE scheduled_at > NOW()")
+          scheduled_count = parse_query_result(result).first
+          stats[:total_scheduled] = (scheduled_count&.dig("count") || scheduled_count&.dig(:count) || 0).to_i
+
+          # failed jobs
+          if conn.table_exists?("solid_queue_failed_jobs")
+            result = conn.execute("SELECT COUNT(*) as count FROM solid_queue_failed_jobs")
+            failed_count = parse_query_result(result).first
+            stats[:total_failed] = (failed_count&.dig("count") || failed_count&.dig(:count) || 0).to_i
+          end
+        rescue => e
+          stats[:error_class] = e.class.name
+          stats[:error_message] = e.message.to_s[0, 500]
+        end
+
+        stats
+      rescue => e
+        {
+          error_class: e.class.name,
+          error_message: e.message.to_s[0, 500]
+        }
+      end
+
+      # --- Delayed::Job collector -------------------------------------------------
+
+      def collect_delayed_job_stats
+        return {} unless defined?(Delayed::Job)
+        return {} unless defined?(ActiveRecord)
+
+        stats = {total_queued: 0, total_busy: 0, queues: {}}
+
+        begin
+          return stats unless ActiveRecord::Base.connection.table_exists?("delayed_jobs")
+
+          # queued jobs
+          queued = Delayed::Job.where("locked_at IS NULL AND attempts < max_attempts").count
+          stats[:total_queued] = queued
+          stats[:queues]["default"] = {queued: queued, busy: 0, scheduled: 0, retries: 0}
+
+          # busy jobs
+          busy = Delayed::Job.where("locked_at IS NOT NULL AND locked_by IS NOT NULL").count
+          stats[:total_busy] = busy
+          stats[:queues]["default"][:busy] = busy
+
+          # failed jobs
+          failed = Delayed::Job.where("attempts >= max_attempts").count
+          stats[:total_failed] = failed
+        rescue => e
+          stats[:error_class] = e.class.name
+          stats[:error_message] = e.message.to_s[0, 500]
+        end
+
+        stats
+      rescue => e
+        {
+          error_class: e.class.name,
+          error_message: e.message.to_s[0, 500]
+        }
+      end
+
+      # --- GoodJob collector ------------------------------------------------------
+
+      def collect_good_job_stats
+        return {} unless defined?(GoodJob)
+        return {} unless defined?(ActiveRecord)
+        return {} unless ActiveRecord::Base.respond_to?(:connected?) && ActiveRecord::Base.connected?
+
+        stats = {total_queued: 0, total_busy: 0, queues: {}}
+
+        begin
+          conn = ActiveRecord::Base.connection
+          return stats unless conn.respond_to?(:table_exists?) && conn.table_exists?("good_jobs")
+
+          # queued
+          result = conn.execute("SELECT queue_name, COUNT(*) as count FROM good_jobs WHERE finished_at IS NULL GROUP BY queue_name")
+          parse_query_result(result).each do |row|
+            queue_name = (row["queue_name"] || row[:queue_name] || "default").to_s
+            count = (row["count"] || row[:count] || 0).to_i
+            stats[:queues][queue_name] = {queued: count, busy: 0, scheduled: 0, retries: 0}
+            stats[:total_queued] += count
+          end
+
+          # busy
+          result = conn.execute("SELECT queue_name, COUNT(*) as count FROM good_jobs WHERE finished_at IS NULL AND performed_at IS NOT NULL GROUP BY queue_name")
+          parse_query_result(result).each do |row|
+            queue_name = (row["queue_name"] || row[:queue_name] || "default").to_s
+            count = (row["count"] || row[:count] || 0).to_i
+            stats[:queues][queue_name] ||= {queued: 0, busy: 0, scheduled: 0, retries: 0}
+            stats[:queues][queue_name][:busy] = count
+            stats[:total_busy] += count
+          end
+
+          # scheduled
+          result = conn.execute("SELECT COUNT(*) as count FROM good_jobs WHERE scheduled_at > NOW()")
+          scheduled_count = parse_query_result(result).first
+          stats[:total_scheduled] = (scheduled_count&.dig("count") || scheduled_count&.dig(:count) || 0).to_i
+
+          # failed
+          result = conn.execute("SELECT COUNT(*) as count FROM good_jobs WHERE finished_at IS NOT NULL AND error IS NOT NULL")
+          failed_count = parse_query_result(result).first
+          stats[:total_failed] = (failed_count&.dig("count") || failed_count&.dig(:count) || 0).to_i
+        rescue => e
+          stats[:error_class] = e.class.name
+          stats[:error_message] = e.message.to_s[0, 500]
+        end
+
+        stats
+      rescue => e
+        {
+          error_class: e.class.name,
+          error_message: e.message.to_s[0, 500]
+        }
+      end
+
+      # --- Shared helpers ---------------------------------------------------------
+
+      def parse_query_result(result)
+        if result.respond_to?(:each)
+          if result.respond_to?(:values)
+            begin
+              columns = begin
+                result.fields
+              rescue
+                result.column_names
+              end
+            rescue
+              []
+            end
+            result.values.map do |row|
+              columns.each_with_index.each_with_object({}) do |(col, idx), hash|
+                hash[col.to_s] = row[idx]
+                hash[col.to_sym] = row[idx]
+              end
+            end
+          elsif result.is_a?(Array)
+            result
+          else
+            result.to_a
+          end
+        else
+          []
+        end
+      rescue
+        []
+      end
+
+      def safe_app_name
+        if defined?(Rails) && Rails.respond_to?(:application) && Rails.application
+          begin
+            Rails.application.class.module_parent_name
+          rescue
+            Rails.application.class.name
+          end
+        end
+      rescue
+        nil
+      end
+
+      def process_hostname
+        if defined?(ProcessInfo)
+          begin
+            ProcessInfo.safe_hostname
+          rescue
+            default_hostname
+          end
+        else
+          default_hostname
+        end
+      rescue
+        default_hostname
+      end
+
+      def default_hostname
+        require "socket"
+        Socket.gethostname
+      rescue
+        "unknown"
+      end
+
+      def safe_collect
+        yield
+      rescue => e
+        {error_class: e.class.name, error_message: e.message.to_s[0, 500]}
+      end
+    end
+  end
+end

data/lib/dead_bro/collectors/network.rb

@@ -0,0 +1,252 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "rbconfig"
+
+module DeadBro
+  module Collectors
+    # Network collector exposes best-effort rx/tx byte counters and
+    # per-interval rates for Linux systems via /proc/net/dev.
+    module Network
+      module_function
+
+      SAMPLE_KEY = "network"
+
+      def collect
+        if linux? && File.readable?("/proc/net/dev")
+          current = read_interfaces_linux
+        elsif macos?
+          current = read_interfaces_macos
+        else
+          return {available: false}
+        end
+
+        return {available: false} if current.empty?
+
+        now = current_time
+        prev = SampleStore.load(SAMPLE_KEY)
+        SampleStore.save(SAMPLE_KEY, {"timestamp" => now, "interfaces" => current})
+
+        # Filter to keep only the top interface by total activity (rx + tx)
+        top_interface = current.max_by do |_, data|
+          (data["rx_bytes"] || 0) + (data["tx_bytes"] || 0)
+        end
+
+        # current is a Hash: { "eth0" => { ... }, ... }
+        # top_interface is an Array: ["eth0", { ... }] or nil
+
+        filtered_current = {}
+        filtered_current[top_interface[0]] = top_interface[1] if top_interface
+
+        # Save *all* current interfaces to store for continuity,
+        # but only report the top one to the backend.
+        # Actually, if we switch top interface, we need history for the new one.
+        # So we should save all, but only return one.
+
+        {
+          available: true,
+          interfaces: build_interface_stats(prev, filtered_current, now)
+        }
+      rescue => e
+        {
+          error_class: e.class.name,
+          error_message: e.message.to_s[0, 500]
+        }
+      end
+
+      def linux?
+        host_os = RbConfig::CONFIG["host_os"].to_s.downcase
+        host_os.include?("linux")
+      rescue
+        false
+      end
+
+      def macos?
+        host_os = RbConfig::CONFIG["host_os"].to_s.downcase
+        host_os.include?("darwin")
+      rescue
+        false
+      end
+
+      def current_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      rescue
+        Time.now.to_f
+      end
+
+      def ignore_interfaces
+        if DeadBro.configuration.respond_to?(:interfaces_ignore)
+          DeadBro.configuration.interfaces_ignore || default_ignore
+        else
+          default_ignore
+        end
+      rescue
+        default_ignore
+      end
+
+      def default_ignore
+        %w[lo lo0 docker0]
+      end
+
+      def read_interfaces_linux
+        ignored = ignore_interfaces
+        interfaces = {}
+
+        File.foreach("/proc/net/dev") do |line|
+          next unless line.include?(":")
+
+          name, data = line.split(":", 2)
+          name = name.strip
+          next if ignored.include?(name)
+
+          fields = data.split
+          # /proc/net/dev format:
+          # Inter-|   Receive                                                |  Transmit
+          #  face |bytes    packets errs drop fifo frame compressed multicast|bytes    packets errs drop fifo colls carrier compressed
+          rx_bytes = begin
+            Integer(fields[0])
+          rescue
+            nil
+          end
+          tx_bytes = begin
+            Integer(fields[8])
+          rescue
+            nil
+          end
+          next unless rx_bytes && tx_bytes
+
+          interfaces[name] = {
+            "rx_bytes" => rx_bytes,
+            "tx_bytes" => tx_bytes
+          }
+        end
+
+        interfaces
+      rescue
+        {}
+      end
+
+      def read_interfaces_macos
+        ignored = ignore_interfaces
+        interfaces = {}
+
+        # netstat -ib output format (simplified):
+        # Name  Mtu   Network       Address            Ipkts Ierrs    Ibytes    Opkts Oerrs     Obytes  Coll
+        # lo0   16384 <Link#1>                        309756     0  49057632   309756     0   49057632     0
+        # en0   1500  <Link#4>    88:66:5a:00:22:11  2685232     0 3123456789  1501234     0 234567890     0
+
+        output = `netstat -ib`
+        output.each_line do |line|
+          fields = line.split
+          next if fields.size < 10 # heuristic check for header or malformed line
+
+          name = fields[0]
+          fields[2]
+          network = fields[2]
+
+          # We only care about lines with <Link#...> which contain the byte counters
+          next unless network && network.start_with?("<Link#")
+          next if ignored.include?(name)
+
+          # Header columns: Name(0) Mtu(1) Network(2) Address(3) Ipkts(4) Ierrs(5) Ibytes(6) Opkts(7) Oerrs(8) Obytes(9) Coll(10)
+          # Note: Address column might be missing if no MAC address (like lo0), checking field alignment
+          # netstat -ib alignment is tricky, sometimes space separated.
+          # Assuming standard output where <Link#..> is present:
+
+          # For <Link#...> lines:
+          # Name  Mtu   Network    Address            Ipkts Ierrs Ibytes    ...
+          # en0   1500  <Link#4>   88:66:5a:00:22:11  ...   ...   bytes(6)  ... bytes(9)
+          # lo0   16384 <Link#1>                      ...   ...   bytes(5?) -> No address column for lo0 link row?
+          # Let's re-verify netstat -ib output.
+          # Actually "Address" column exists for links, usually MAC address for en0, empty/implied for lo0?
+          # Wait, regex is safer.
+
+          # Try to identify based on identifying the Network column being <Link...>
+
+          # Usually:
+          # fields[0] = Name
+          # ...
+          # fields[2] = <Link#...>
+          # ...
+          # We need to find Ibytes and Obytes.
+          # If Address is present (MAC), Ibytes is at index 6, Obytes at 9.
+          # If Address is NOT present (?), indices shift?
+          # Actually netstat -ib usually aligns content.
+
+          # Let's count from the end?
+          # Typical line: en0 1500 <Link#4> 88:66:5a:... 2685232 0 3123456789 1501234 0 234567890 0
+          # fields: [en0, 1500, <Link#4>, MAC, Ipkts, Ierrs, Ibytes, Opkts, Oerrs, Obytes, Coll] -> 11 fields
+          # Ibytes = 6, Obytes = 9
+
+          # lo0 line: lo0 16384 <Link#1> 309756 0 49057632 309756 0 49057632 0
+          # fields: [lo0, 16384, <Link#1>, Ipkts, Ierrs, Ibytes, Opkts, Oerrs, Obytes, Coll] -> 10 fields? Address missing?
+          # Yes, lo0 often has no address in Link row.
+          # Ibytes = 5, Obytes = 8
+
+          # Logic:
+          # if fields[3] looks like a MAC address, use 6 and 9.
+          # else (assuming it's Ipkts), use 5 and 8.
+
+          # Actually, Ipkts is always an integer. MAC is xx:xx:xx...
+
+          idx_ibytes = 6
+          idx_obytes = 9
+
+          if /^\d+$/.match?(fields[3]) # Field 3 is Ipkts (integer) -> Address column missing
+            idx_ibytes = 5
+            idx_obytes = 8
+          end
+
+          rx_bytes = begin
+            Integer(fields[idx_ibytes])
+          rescue
+            nil
+          end
+          tx_bytes = begin
+            Integer(fields[idx_obytes])
+          rescue
+            nil
+          end
+
+          next unless rx_bytes && tx_bytes
+
+          interfaces[name] = {
+            "rx_bytes" => rx_bytes,
+            "tx_bytes" => tx_bytes
+          }
+        end
+
+        interfaces
+      rescue
+        {}
+      end
+
+      def build_interface_stats(prev, current, now)
+        prev_ts = prev && prev["timestamp"]
+        elapsed = prev_ts ? (now - prev_ts.to_f) : nil
+
+        current.map do |name, data|
+          prev_data = prev && prev["interfaces"] && prev["interfaces"][name]
+          rx_rate = tx_rate = nil
+
+          if elapsed && elapsed > 0 && prev_data
+            rx_delta = data["rx_bytes"] - prev_data["rx_bytes"].to_i
+            tx_delta = data["tx_bytes"] - prev_data["tx_bytes"].to_i
+            rx_rate = (rx_delta / elapsed.to_f).round(2) if rx_delta >= 0
+            tx_rate = (tx_delta / elapsed.to_f).round(2) if tx_delta >= 0
+          end
+
+          {
+            name: name,
+            rx_bytes: data["rx_bytes"],
+            tx_bytes: data["tx_bytes"],
+            rx_bytes_per_s: rx_rate,
+            tx_bytes_per_s: tx_rate
+          }
+        end
+      rescue
+        []
+      end
+    end
+  end
+end