sidekiq-datadog-monitor 0.3.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a6e0e510899e6b233d1bb1d275f9d708c851567176894074ce1d83bc45f21a7a
-  data.tar.gz: af4b6268dec084b4dbc9e3307fdeab2015a57544aaa51a058e9fc8e64f25434c
+  metadata.gz: 916092eba3887bd8efc9c819817ac2f971cafd3767fe5c801e412ba4e2eefa74
+  data.tar.gz: 5c110f7a204af943aadc0d6c8d7719d58bb98019f6d9fd1dd37df84b0e3c443d
 SHA512:
-  metadata.gz: 3ad0a766f6d1648815271cd574da145e74599442c0611f8892ffc1c3f650ebf1375173e8600e9e7a30a4a4031c3eee3822d04e80a925fe663a214ccf908c4bf5
-  data.tar.gz: 1291ed9981ad48deb2ccb0e52226f87f199fac0b7da85d892ee227eb34ea2dfa4aa366c9197a457ba013d9618edba5df64a02c1e71b70425eb0317fb5476c28a
+  metadata.gz: 506f09c420c679a9220d7da7805732b723c85fa3ccc93fd91fa94ff070f31d996ba150ef9f7558e86e02f86c38b8f2c1527bac5bf8ffd03fd985dff44272445f
+  data.tar.gz: 8e7a1ba96cec2599361ea43f0a770d7cd1c6da779a22cf85b3b80d621067df336ffa4a383a9cee3811bbc86018e16c4f74b7924bf63e3f3a279208431be238e1

data/README.md

@@ -25,27 +25,23 @@ To start sending metrics
 
 ```ruby
 # Import the library
-require 'sidekiq/datadog/monitor/data'
+require 'sidekiq/datadog/monitor'
 
-# Initiate a Sidekiq::Datadog::Monitor client instance.
-Sidekiq::Datadog::Monitor::Data.initialize!(
+# Configure Sidekiq::Datadog::Monitor.
+Sidekiq::Datadog::Monitor.configure!(
   agent_host: 'localhost',
   agent_port: 8125,
-  queue: 'queue name',
   tags: ['env:production', 'product:product_name'], # optional
-  cron: "*/30 * * * *" # default: "*/1 * * * *",
-  batch: false # optional, default: false
 )
 ```
-`agent_host` and `agent_port` instantiate DogStatsD client
+ - `agent_host` and `agent_port` instantiate DogStatsD client
+ - `tags` additional tags to be added for every metrics
 
-`queue` setting for background job that will gather and send Sidekiq metrics
+## How it works
 
-`tags` tags for datadog metrics
-
-`cron` - schedule settings for background job that will gather and send Sidekiq metrics
-
-`batch` turns on sending DD metrics in batches. Make sure you don't have too many queues before enabling this option. The message with all tags must fit into [8KB of default DataDog buffer](https://docs.datadoghq.com/developers/dogstatsd/high_throughput/#enable-buffering-on-your-client) size.
+Gem using sidekiq lifecycle events to operate and send metrics to datadog. 
+Heartbeat event is used to send metrics every 5 seconds. Heartbeat event happens in every sidekiq process so metrics 
+will be sent multiple times but will be aggregated on datadog.
 
 
 ## Development

data/lib/sidekiq/datadog/monitor/metrics_sender.rb

@@ -0,0 +1,41 @@
+require 'sidekiq/api'
+
+module Sidekiq
+  module Datadog
+    module Monitor
+      class MetricsSender
+        attr_reader :statsd, :tags_builder
+        def initialize(statsd, tags_builder)
+          @statsd = statsd
+          @tags_builder = tags_builder
+        end
+
+        def send_metrics
+          Sidekiq::Stats.new.queues.each_pair do |queue_name, size|
+            post_queue_stats(statsd, queue_name, size)
+          end
+          Sidekiq::ProcessSet.new.each do |process|
+            post_process_stats(process)
+          end
+        end
+
+        protected
+
+        def post_queue_stats(statsd, queue_name, size)
+          latency = Sidekiq::Queue.new(queue_name).latency
+          tags = tags_builder.build(queue_name: queue_name)
+
+          statsd.gauge('sidekiq.queue.size', size, tags: tags)
+          statsd.gauge('sidekiq.queue.latency', latency, tags: tags)
+        end
+
+        def post_process_stats(process)
+          utilization = process['busy'] / process['concurrency'].to_f
+          tags = tags_builder.build(process_id: process['identity'], process_tag: process['tag'])
+
+          statsd.gauge('sidekiq.process.utilization', utilization, tags: tags)
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/datadog/monitor/tag_builder.rb

@@ -0,0 +1,24 @@
+module Sidekiq
+  module Datadog
+    module Monitor
+      class TagBuilder
+        SPECIAL_SYMBOLS = /[\/\.:]/i.freeze
+        def initialize(common_tags)
+          @common_tags = common_tags
+        end
+
+        def build(tags_hash)
+          custom_tags = tags_hash.map { |key, value| [key, value] if value.to_s != '' }.compact
+          custom_tags = custom_tags.map { |key, value| "#{key}:#{normalize_value(value)}" }
+          custom_tags + @common_tags
+        end
+
+        protected
+
+        def normalize_value(value)
+          value.to_s.gsub(SPECIAL_SYMBOLS, '_')
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/datadog/monitor/version.rb

@@ -1,7 +1,7 @@
 module Sidekiq
   module Datadog
     module Monitor
-      VERSION = '0.3.0'.freeze
+      VERSION = '1.0.0'.freeze
     end
   end
 end

data/lib/sidekiq/datadog/monitor.rb

@@ -1,12 +1,67 @@
-require 'sidekiq/api'
-require 'sidekiq-scheduler'
-require 'sidekiq/datadog/monitor/data'
-require 'sidekiq/datadog/monitor/metrics_worker'
+require 'datadog/statsd'
+require 'sidekiq/datadog/monitor/metrics_sender'
+require 'sidekiq/datadog/monitor/tag_builder'
 
 module Sidekiq
   module Datadog
     module Monitor
       class Error < StandardError; end
+
+      class << self
+        attr_accessor :agent_port, :agent_host, :tags_builder, :statsd, :sender
+
+        def configure!(options)
+          raise Sidekiq::Datadog::Monitor::Error, "Can't configure two times" if configured?
+
+          @agent_host, @agent_port = options.fetch_values(:agent_host, :agent_port)
+          @tags_builder = Sidekiq::Datadog::Monitor::TagBuilder.new(options[:tags] || [])
+
+          add_sidekiq_listeners
+        rescue KeyError => e
+          raise Sidekiq::Datadog::Monitor::Error, "Required param is missing: #{e.message}"
+        end
+
+        def configured?
+          agent_host && agent_port
+        end
+
+        def initialize!
+          @statsd = ::Datadog::Statsd.new(agent_host, agent_port)
+          @sender = Sidekiq::Datadog::Monitor::MetricsSender.new(statsd, tags_builder)
+        end
+
+        def send_metrics
+          sender.call
+        end
+
+        def shutdown!
+          statsd.close
+        end
+
+
+        private
+
+        def reset!
+          @agent_host = nil
+          @agent_port = nil
+          @statsd = nil
+          @sender = nil
+        end
+
+        def add_sidekiq_listeners
+          Sidekiq.configure_server do |config|
+            config.on(:startup) do
+              Sidekiq::Datadog::Monitor.initialize!
+            end
+            config.on(:heartbeat) do
+              Sidekiq::Datadog::Monitor.send_metrics
+            end
+            config.on(:shutdown) do
+              Sidekiq::Datadog::Monitor.shutdown!
+            end
+          end
+        end
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-datadog-monitor
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 1.0.0
 platform: ruby
 authors:
 - aleksa_castle
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-06-22 00:00:00.000000000 Z
+date: 2023-02-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -108,20 +108,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.2.1
-- !ruby/object:Gem::Dependency
-  name: sidekiq-scheduler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: dogstatsd-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -148,8 +134,8 @@ files:
 - bin/console
 - bin/setup
 - lib/sidekiq/datadog/monitor.rb
-- lib/sidekiq/datadog/monitor/data.rb
-- lib/sidekiq/datadog/monitor/metrics_worker.rb
+- lib/sidekiq/datadog/monitor/metrics_sender.rb
+- lib/sidekiq/datadog/monitor/tag_builder.rb
 - lib/sidekiq/datadog/monitor/version.rb
 homepage: https://github.com/matic-insurance/sidekiq-datadog-monitor
 licenses:

data/lib/sidekiq/datadog/monitor/data.rb

@@ -1,34 +0,0 @@
-module Sidekiq
-  module Datadog
-    module Monitor
-      class Data
-        class << self
-          attr_reader :agent_port, :agent_host, :tags, :env, :queue, :cron
-
-          def initialize!(options)
-            @agent_port, @agent_host, @queue = options.fetch_values(:agent_port, :agent_host, :queue)
-            @tags = options[:tags] || []
-            @cron = options[:cron] || '*/1 * * * *'
-
-            Sidekiq.configure_server do |config|
-              SidekiqScheduler::Scheduler.dynamic = true
-
-              config.on(:startup) do
-                start
-              end
-            end
-          rescue StandardError => e
-            raise Sidekiq::Datadog::Monitor::Error, e.message
-          end
-
-          private
-
-          def start
-            Sidekiq.set_schedule('send_metrics',
-                                 { 'cron' => cron, 'class' => 'Sidekiq::Datadog::Monitor::MetricsWorker', 'queue' => queue })
-          end
-        end
-      end
-    end
-  end
-end

data/lib/sidekiq/datadog/monitor/metrics_worker.rb

@@ -1,41 +0,0 @@
-require 'sidekiq/datadog/monitor/data'
-require 'datadog/statsd'
-
-module Sidekiq
-  module Datadog
-    module Monitor
-      class MetricsWorker
-        include Sidekiq::Worker
-
-        sidekiq_options retry: false
-
-        def perform
-          statsd = ::Datadog::Statsd.new(Data.agent_host, Data.agent_port)
-          send_metrics(statsd)
-          statsd.close
-        end
-
-        private
-
-        def send_metrics(statsd)
-          Sidekiq::Stats.new.queues.each_pair do |queue_name, size|
-            post_queue_size(statsd, queue_name, size)
-
-            post_queue_latency(statsd, queue_name)
-          end
-        end
-
-        def post_queue_size(statsd, queue_name, size)
-          statsd.gauge('sidekiq.queue.size', size,
-                       tags: ["queue_name:#{queue_name}"].concat(Data.tags))
-        end
-
-        def post_queue_latency(statsd, queue_name)
-          latency = Sidekiq::Queue.new(queue_name).latency
-          statsd.gauge('sidekiq.queue.latency', latency,
-                       tags: ["queue_name:#{queue_name}"].concat(Data.tags))
-        end
-      end
-    end
-  end
-end