dead_bro 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64333e774f85e0505014b6a0a6170e6721c0be30f1d3103b214fabdc7faacd8f
-  data.tar.gz: 82d6fc30651fcc2ee54904c6dff7b7cc36bd56c1a27a58ee726429c8b592b50a
+  metadata.gz: 7f4e8376defc769119dd530481b5742580e2c296c09c065bd9f1e6f6d6d687b2
+  data.tar.gz: ca5caacda90af41400df74709567c70001aac71f83a174b3da3976dc5d673d3a
 SHA512:
-  metadata.gz: f7fc637b60a2990c3e3cfd6c39a5adbbebde198b11c1da5ff1127199ee05e900c6b971bccc740b1552f53156717511c2244dc7e70f736ac2b2110a2e0291d577
-  data.tar.gz: 1bed47b5f0d4dabc1e851631c00701802207769280b82d2809afd7cf75fa3e5be5dfaf5940ed853234dfdf659911512f56a383020b2bb027bce6eea8d1288637
+  metadata.gz: 241f825cf7696926f57fbc30ef59a0fc2e235173f4b6ad72ce4d6eecab393b77c037bd6deab20d780fe275289cac01955562bfb62a70e331e5294b4b607ea305
+  data.tar.gz: '0988d794e49eb4a825ff7936e00cd41ad98b73b67d45126b1ff537eef47ed1f17562c080f82091ae455a12e261a92ead0aea1a0bbed1ade7468f5635693e42b6'

data/README.md

@@ -262,6 +262,132 @@ DeadBro automatically tracks ActiveJob background jobs when ActiveJob is availab
 - `message` - Exception message (for failed jobs)
 - `backtrace` - Exception backtrace (for failed jobs)
 
+## Control Plane Metrics (Queues, DB, Process, System)
+
+DeadBro includes a lightweight **control plane metrics** job that runs periodically (by default once per minute via `DeadBro::JobQueueMonitor`) and sends a single JSON payload summarizing:
+
+- **Sidekiq / job queues**: global stats (`processed`, `failed`, `enqueued`, `scheduled_size`, `retry_size`, `dead_size`, `workers_size`, `processes_size`), and per-queue entries with `name`, `size`, and `latency_s`.
+- **Database (best effort)**: connection pool stats and a simple `ping_ms` latency when `ActiveRecord` is available and connected.
+- **Process / Rails**: `pid`, `hostname`, uptime, Ruby/Rails versions, environment, GC stats, RSS (`rss_bytes`), thread and file descriptor counts (on Linux).
+- **System (Linux best effort)**: CPU percentage over the last interval (normalised 0–100), memory used/total/available, plus filesystem and network summaries.
+
+Everything is **best effort** and designed to be **safe and low overhead**:
+
+- Collection never raises; failures are reported as `{error_class, error_message}` under the respective section key.
+- No sensitive data is sent (no job arguments, env vars, CLI args, or full SQL text in these control-plane metrics).
+- CPU and network *rates* require two samples; on the first run you may see `nil` for `cpu_pct` or network `*_bytes_per_s` fields until a second sample is available.
+
+### Configuration
+
+You can enable or disable individual collectors and tune basic options via the standard `DeadBro.configure` block:
+
+```ruby
+DeadBro.configure do |config|
+  # Enable the periodic job queue monitor (disabled by default)
+  config.job_queue_monitoring_enabled = true
+
+  # Enable best-effort collectors (all default to false)
+  config.enable_db_stats      = true   # ActiveRecord pool + ping latency
+  config.enable_process_stats = true   # pid, hostname, RSS, GC, threads, fds
+  config.enable_system_stats  = true   # CPU%, memory, disk, network
+
+  # Filesystem paths to report disk usage for (default: ["/"])
+  config.disk_paths = ["/", "/var"]
+
+  # Network interfaces to ignore when computing rx/tx stats
+  config.interfaces_ignore = %w[lo docker0]
+end
+```
+
+### Example Payload Shape
+
+The control plane job sends a single JSON payload roughly shaped like:
+
+```json
+{
+  "ts": "2025-01-01T12:00:00Z",
+  "app_name": "MyApp",
+  "env": "production",
+  "host": "app-1",
+  "pid": 12345,
+  "versions": {
+    "ruby": "3.1.2",
+    "rails": "7.1.0",
+    "sidekiq": "7.3.0"
+  },
+  "queue_system": "sidekiq",
+  "sidekiq": {
+    "processed": 1000,
+    "failed": 5,
+    "enqueued": 42,
+    "scheduled_size": 3,
+    "retry_size": 2,
+    "dead_size": 1,
+    "workers_size": 4,
+    "processes_size": 2,
+    "memory_rss_bytes": 123456789,
+    "queues": [
+      {"name": "default", "size": 10, "latency_s": 0.5}
+    ]
+  },
+  "db": {
+    "available": true,
+    "pool": {
+      "size": 5,
+      "connections": 3,
+      "busy": 1,
+      "num_waiting": 0
+    },
+    "ping_ms": 2.1
+  },
+  "process": {
+    "pid": 12345,
+    "hostname": "app-1",
+    "uptime_s": 3600.5,
+    "rss_bytes": 123456789,
+    "thread_count": 20,
+    "fd_count": 128,
+    "gc": {
+      "heap_live_slots": 123_456,
+      "heap_free_slots": 12_345,
+      "total_allocated_objects": 1_234_567,
+      "major_gc_count": 10,
+      "minor_gc_count": 50
+    }
+  },
+  "system": {
+    "cpu_pct": 12.3,
+    "mem_used_bytes": 987654321,
+    "mem_total_bytes": 2147483648,
+    "mem_available_bytes": 1153433600,
+    "disk": {
+      "paths": [
+        {
+          "path": "/",
+          "disk_total_bytes": 107374182400,
+          "disk_free_bytes": 53687091200,
+          "disk_available_bytes": 53687091200
+        }
+      ]
+    },
+    "net": {
+      "available": true,
+      "interfaces": [
+        {
+          "name": "eth0",
+          "rx_bytes": 123456,
+          "tx_bytes": 654321,
+          "rx_bytes_per_s": 1000.0,
+          "tx_bytes_per_s": 500.0
+        }
+      ]
+    }
+  }
+}
+```
+
+Not all fields will be present in all environments; unsupported or unavailable metrics may be `null` or omitted, and any hard failures are captured in `error_class` / `error_message` fields per section.
+
 
 ## Development
 

data/lib/dead_bro/cache_subscriber.rb

@@ -49,17 +49,17 @@ module DeadBro
     def self.should_continue_tracking?
       events = Thread.current[THREAD_LOCAL_KEY]
       return false unless events
-      
+
       # Check count limit
       return false if events.length >= MAX_TRACKED_EVENTS
-      
+
       # Check time limit
       start_time = Thread.current[DeadBro::TRACKING_START_TIME_KEY]
       if start_time
         elapsed_seconds = Time.now - start_time
         return false if elapsed_seconds >= DeadBro::MAX_TRACKING_DURATION_SECONDS
       end
-      
+
       true
     end
 

data/lib/dead_bro/client.rb

@@ -18,7 +18,7 @@ module DeadBro
 
       # Check sampling rate - skip if not selected for sampling
       return unless @configuration.should_sample?
-      
+
       # Check circuit breaker before making request
       if @circuit_breaker && @configuration.circuit_breaker_enabled
         if @circuit_breaker.state == :open
@@ -37,7 +37,7 @@ module DeadBro
       nil
     end
 
-    def post_job_stats(payload)
+    def post_monitor_stats(payload)
       return if @configuration.api_key.nil?
       return unless @configuration.enabled
       return unless @configuration.job_queue_monitoring_enabled
@@ -55,7 +55,7 @@ module DeadBro
       end
 
       # Make the HTTP request (async) to jobs endpoint
-      make_job_stats_request(payload, @configuration.api_key)
+      make_monitor_request(payload, @configuration.api_key)
 
       nil
     end
@@ -122,10 +122,10 @@ module DeadBro
       nil
     end
 
-    def make_job_stats_request(payload, api_key)
+    def make_monitor_request(payload, api_key)
       use_staging = ENV["USE_STAGING_ENDPOINT"] && !ENV["USE_STAGING_ENDPOINT"].empty?
-      production_url = use_staging ? "https://deadbro.aberatii.com/apm/v1/jobs" : "https://www.deadbro.com/apm/v1/jobs"
-      endpoint_url = @configuration.ruby_dev ? "http://localhost:3100/apm/v1/jobs" : production_url
+      production_url = use_staging ? "https://deadbro.aberatii.com/apm/v1/monitor" : "https://www.deadbro.com/apm/v1/monitor"
+      endpoint_url = @configuration.ruby_dev ? "http://localhost:3100/apm/v1/monitor" : production_url
       uri = URI.parse(endpoint_url)
       http = Net::HTTP.new(uri.host, uri.port)
       http.use_ssl = (uri.scheme == "https")

data/lib/dead_bro/collectors/database.rb

@@ -0,0 +1,126 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+module DeadBro
+  module Collectors
+    # Database collector provides lightweight, best-effort information
+    # about the current ActiveRecord connection pool and a simple ping
+    # latency measurement.
+    module Database
+      module_function
+
+      def collect
+        return {disabled: true} unless db_enabled?
+        return {available: false} unless defined?(::ActiveRecord)
+
+        base = ::ActiveRecord::Base
+        return {available: false} unless base.respond_to?(:connection_pool) && base.connection_pool
+
+        pool = safe_connection_pool(base)
+
+        {
+          available: true,
+          pool: pool_stats(pool),
+          ping_ms: ping_ms(base)
+        }
+      rescue => e
+        {
+          error_class: e.class.name,
+          error_message: e.message.to_s[0, 500]
+        }
+      end
+
+      def db_enabled?
+        DeadBro.configuration.respond_to?(:enable_db_stats) &&
+          DeadBro.configuration.enable_db_stats
+      rescue
+        false
+      end
+
+      def safe_connection_pool(base)
+        if base.respond_to?(:connection_pool)
+          base.connection_pool
+        elsif base.respond_to?(:connection_handler)
+          begin
+            base.connection_handler.retrieve_connection_pool(base)
+          rescue
+            nil
+          end
+        end
+      rescue
+        nil
+      end
+
+      def pool_stats(pool)
+        return {} unless pool
+
+        {
+          size: begin
+            safe_integer(pool.size)
+          rescue
+            nil
+          end,
+          connections: begin
+            safe_integer(pool.connections.size)
+          rescue
+            nil
+          end,
+          busy: begin
+            safe_integer(pool.respond_to?(:busy) ? pool.busy : nil)
+          rescue
+            nil
+          end,
+          dead: begin
+            safe_integer(pool.respond_to?(:dead) ? pool.dead : nil)
+          rescue
+            nil
+          end,
+          num_waiting: begin
+            safe_integer(pool.respond_to?(:num_waiting) ? pool.num_waiting : nil)
+          rescue
+            nil
+          end,
+          automatic_reconnect: pool.respond_to?(:automatic_reconnect) ? !!pool.automatic_reconnect : nil
+        }
+      rescue
+        {}
+      end
+
+      def ping_ms(base)
+        started = current_time
+        base.connection_pool.with_connection do |conn|
+          sql = case conn.adapter_name.to_s.downcase
+          when /mysql/
+            "SELECT 1"
+          when /sqlite/
+            "SELECT 1"
+          else
+            "SELECT 1"
+          end
+          conn.select_value(sql)
+        end
+        elapsed_ms(started)
+      rescue
+        nil
+      end
+
+      def current_time
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      rescue
+        Time.now.to_f
+      end
+
+      def elapsed_ms(started)
+        ((current_time - started) * 1000.0).round(2)
+      rescue
+        nil
+      end
+
+      def safe_integer(value)
+        Integer(value)
+      rescue
+        nil
+      end
+    end
+  end
+end

data/lib/dead_bro/collectors/filesystem.rb

@@ -0,0 +1,94 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "rbconfig"
+
+module DeadBro
+  module Collectors
+    # Filesystem collector exposes disk usage information using a best-effort
+    # approach. It prefers Ruby or Sys::Filesystem APIs when available and
+    # falls back to parsing `df` output.
+    module Filesystem
+      module_function
+
+      def collect
+        paths = disk_paths
+        return {paths: []} if paths.nil? || paths.empty?
+
+        {
+          paths: paths.map { |path| stats_for_path(path) }.compact
+        }
+      rescue => e
+        {
+          error_class: e.class.name,
+          error_message: e.message.to_s[0, 500]
+        }
+      end
+
+      def disk_paths
+        if DeadBro.configuration.respond_to?(:disk_paths)
+          DeadBro.configuration.disk_paths || ["/"]
+        else
+          ["/"]
+        end
+      rescue
+        ["/"]
+      end
+
+      def stats_for_path(path)
+        if defined?(Sys::Filesystem)
+          sys_filesystem_stats(path)
+        else
+          df_stats(path)
+        end
+      rescue
+        nil
+      end
+
+      def sys_filesystem_stats(path)
+        stat = Sys::Filesystem.stat(path)
+        {
+          path: path,
+          disk_total_bytes: stat.blocks * stat.block_size,
+          disk_free_bytes: stat.blocks_available * stat.block_size,
+          disk_available_bytes: stat.blocks_available * stat.block_size
+        }
+      rescue
+        nil
+      end
+
+      def df_stats(path)
+        # Use POSIX df when available. Output format varies slightly by platform,
+        # but we only depend on total and available in blocks.
+        output = `df -k #{Shellwords.escape(path)} 2>/dev/null`
+        lines = output.to_s.split("\n")
+        return nil if lines.size < 2
+
+        lines[0]
+        data = lines[1]
+        parts = data.split
+        # POSIX df: Filesystem 1K-blocks Used Available Use% Mounted on
+        total_kb = begin
+          Integer(parts[1])
+        rescue
+          nil
+        end
+        avail_kb = begin
+          Integer(parts[3])
+        rescue
+          nil
+        end
+        return nil unless total_kb && avail_kb
+
+        {
+          path: path,
+          disk_total_bytes: total_kb * 1024,
+          disk_free_bytes: avail_kb * 1024,
+          disk_available_bytes: avail_kb * 1024
+        }
+      rescue
+        nil
+      end
+    end
+  end
+end