sidekiq-heartbeat_monitor 1.0.1.4 → 1.0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 114ff3014d1ea216913a8baf3b5d5373059fcbea
-  data.tar.gz: e25b151522f80dc4ece1879a6ed09c1915ef2a2e
+  metadata.gz: a7220abb41d5b0d785ef913cad349184e8e61e10
+  data.tar.gz: ef32dc8543669e7e375de343d6e69433d674d8d5
 SHA512:
-  metadata.gz: 6b794db468711f841ef6a3516bb36b137ac1a563b5ac70fa7e5aab2f4dcaed6cc4e05b72cb57628ae3323716a539bfdcbf79d17948dcd2b1d45d0e0cedd0e1fb
-  data.tar.gz: 59b6100b413ec2399cb473ea9deed3ee5c636dfac6445e05fce6fa201ece914b69f0ae4212e06113aef90b96222ae17665da244427032c91182f93934500e3d0
+  metadata.gz: 45121b850630fa2c1df916c733014d26198506ccfc10315dfd36f9b16551427792a29a78c44d54bf36752c3d4d208bf4e60c332f62f667669a482bcb24f0a4cb
+  data.tar.gz: fd515bb0ab900404ad85e536e0b512a603f75cc5a88d941b0755d553dbfa1ce0bc5c47a6c91b8bc40615f0b83ff37896128c7d20f9858d04bba1419f84937425

data/lib/sidekiq/heartbeat_monitor.rb

@@ -1,29 +1,38 @@
 require "sidekiq"
 require 'sidekiq/heartbeat_monitor/version'
 require 'sidekiq/heartbeat_monitor/config'
+require 'sidekiq/heartbeat_monitor/util'
 require 'sidekiq/heartbeat_monitor/scheduler'
 require 'sidekiq/heartbeat_monitor/worker'
 
 module Sidekiq
   module HeartbeatMonitor
-    def self.configure(*args, &block)
-      Config.configure(*args, &block)
-    end
+    def self.configure(options = {})
+      options = options.symbolize_keys
+      global_options = options.except(:queues)
+
+      @global_config = Config.new(**global_options)
 
-    def self.send_backed_up_alert(*args, &block)
-      Config.send_backed_up_alert(*args, &block)
+      @queue_config = {}
+      options[:queues].to_a.each do |queue_name, queue_options|
+        @queue_config[queue_name.to_s] = Config.new(**global_options.deep_merge(queue_options))
+      end
     end
 
-    def self.send_slowed_down_alert(*args, &block)
-      Config.send_slowed_down_alert(*args, &block)
+    def self.config(queue = nil)
+      return @global_config if queue.blank?
+
+      queue_name = queue.is_a?(String) || queue.is_a?(Symbol) ? queue.to_s : queue.name.to_s
+      @queue_config[queue_name] || @global_config
     end
 
-    def self.send_test!
-      test_queue = Sidekiq::Queue.new('test')
+    def self.send_test!(queue_name = nil)
+      test_queue = Sidekiq::Queue.new(queue_name || 'test')
 
       send_backed_up_alert("Test backed up alert!", test_queue)
       send_slowed_down_alert("Test slowed down alert!", test_queue)
     end
+
   end
 end
 

data/lib/sidekiq/heartbeat_monitor/config.rb

@@ -2,73 +2,96 @@ require 'sidekiq/cron/job'
 module Sidekiq
   module HeartbeatMonitor
     class Config
-      class << self
-        def configure(max_queue_size: nil, on_backed_up: nil, on_slowed_down: nil, dont_repeat_for: nil, slack_notifier_url: nil)
-          @max_queue_size = max_queue_size unless max_queue_size.nil?
+      attr_accessor :max_queue_size, :max_heartbeat_delay
 
-          @dont_repeat_for = dont_repeat_for unless dont_repeat_for.nil?
+      ##
+      # @param max_queue_size [Integer]       The maximum queue size (default: 5000 or SIDEKIQ_MONITOR_MAX_QUEUE_SIZE environment value)
+      # @param on_backed_up [Proc|Array<Proc>]        On backed up run this or these procs
+      # @param on_slowed_down [Proc|Array<Proc>]      On slowed down run this or these procs
+      # @param dont_repeat_for [Integer]     The don't repeat for (optional, default: 5.minutes)
+      # @param slack_notifier_url [String]  The slack notifier url (optional)
+      # @param max_heartbeat_delay [Integer] The maximum heartbeat delay (default: 5 minute, max: 5 days)
+      def initialize(max_queue_size: nil, on_backed_up: nil, on_slowed_down: nil, dont_repeat_for: nil, slack_notifier_url: nil, max_heartbeat_delay: nil)
+        @max_queue_size = max_queue_size ||  ENV.fetch('SIDEKIQ_MONITOR_MAX_QUEUE_SIZE', 5000).to_i
 
-          @on_backed_up = (on_backed_up.is_a?(Enumerable) ? on_backed_up : [on_backed_up]) unless on_backed_up.nil?
-          @on_slowed_down = (on_slowed_down.is_a?(Enumerable) ? on_slowed_down : [on_slowed_down]) unless on_slowed_down.nil?
+        @dont_repeat_for = dont_repeat_for unless dont_repeat_for.nil?
 
-          if slack_notifier_url.present?
-            @notifier = Slack::Notifier.new(slack_notifier_url)
+        @max_heartbeat_delay = max_heartbeat_delay || 5.minutes
 
-            slack_notifier_callback = [-> (msg, queue) { @notifier.ping(msg) }]
+        @on_backed_up = (on_backed_up.is_a?(Enumerable) ? on_backed_up : [on_backed_up]) unless on_backed_up.nil?
+        @on_slowed_down = (on_slowed_down.is_a?(Enumerable) ? on_slowed_down : [on_slowed_down]) unless on_slowed_down.nil?
 
-            @on_backed_up = @on_backed_up.to_a + slack_notifier_callback
-            @on_slowed_down = @on_slowed_down.to_a + slack_notifier_callback
-          end
+        setup_slack_notifier!(slack_notifier_url) if slack_notifier_url.present?
 
-          unless Sidekiq::Cron::Job.find("sidekiq_monitor").present?
-            Sidekiq::Cron::Job.create(
-              name: 'sidekiq_monitor', 
-              cron: '*/15 * * * * *', 
-              klass: Sidekiq::HeartbeatMonitor::Scheduler
-            )
-          end
-        end
+        install_cron_job!
+      end
 
-        def send_test!
-          test_queue = Sidekiq::Queue.new('test')
-          send_backed_up_alert("Test backed up alert!", test_queue)
-          send_slowed_down_alert("Test slowed down alert!", test_queue)
-        end
+      def send_test!
+        test_queue = Sidekiq::Queue.new('test')
+        send_backed_up_alert("Test backed up alert!", test_queue)
+        send_slowed_down_alert("Test slowed down alert!", test_queue)
+      end
 
-        def send_backed_up_alert(message, q)
-          if @on_backed_up.blank?
-            puts ("WARNING: No 'on_backed_up' callback defined for sidekiq-heartbeat_monitor but one of the queues are backed up: #{message}")
-            return
-          end
+      def send_backed_up_alert!(message, q)
+        if @on_backed_up.blank?
+          puts ("WARNING: No 'on_backed_up' callback defined for sidekiq-heartbeat_monitor but one of the queues are backed up: #{message}")
+          return
+        end
 
-          if @dont_repeat_for.nil?
+        if @dont_repeat_for.nil?
+          @on_backed_up.to_a.each { |alert| alert.call(message, q) }
+        else
+          DontRepeatFor.new(@dont_repeat_for, "Sidekiq/HeartbeatMonitor/#{q.name}/send_backed_up_alert") do
             @on_backed_up.to_a.each { |alert| alert.call(message, q) }
-          else
-            DontRepeatFor.new(@dont_repeat_for, "Sidekiq/HeartbeatMonitor/#{q.name}/send_backed_up_alert") do
-              @on_backed_up.to_a.each { |alert| alert.call(message, q) }
-            end
           end
         end
+      end
 
-        def send_slowed_down_alert(message, q)
-          if @on_slowed_down.blank?
-            puts ("WARNING: No 'on_slowed_down' callback defined for sidekiq-heartbeat_monitor but one of the queues are backed up: #{message}")
-            return
-          end
+      def send_slowed_down_alert!(message, q)
+        if @on_slowed_down.blank?
+          puts ("WARNING: No 'on_slowed_down' callback defined for sidekiq-heartbeat_monitor but one of the queues are backed up: #{message}")
+          return
+        end
 
-          if @dont_repeat_for.nil?
+        if @dont_repeat_for.nil?
+          @on_slowed_down.to_a.each { |alert| alert.call(message, q) }
+        else
+          DontRepeatFor.new(@dont_repeat_for, "Sidekiq/HeartbeatMonitor/#{q.name}/send_slowed_down_alert") do
             @on_slowed_down.to_a.each { |alert| alert.call(message, q) }
-          else
-            DontRepeatFor.new(@dont_repeat_for, "Sidekiq/HeartbeatMonitor/#{q.name}/send_slowed_down_alert") do
-              @on_slowed_down.to_a.each { |alert| alert.call(message, q) }
-            end
           end
         end
+      end
+
+      private
+
+      def setup_slack_notifier!(slack_notifier_url)
+        @notifier = Slack::Notifier.new(slack_notifier_url)
+
+        slack_notifier_callback = [-> (msg, queue) { @notifier.ping(msg) }]
+
+        @on_backed_up = @on_backed_up.to_a + slack_notifier_callback
+        @on_slowed_down = @on_slowed_down.to_a + slack_notifier_callback
+      end
+
+      def install_cron_job!
+        job = Sidekiq::Cron::Job.find("sidekiq_monitor")
+
+        target_job = Sidekiq::Cron::Job.new(
+          name: 'sidekiq_monitor', 
+          cron: '* * * * *', 
+          klass: Sidekiq::HeartbeatMonitor::Scheduler
+        )
 
-        def max_queue_size
-          @max_queue_size || ENV.fetch('SIDEKIQ_MONITOR_MAX_QUEUE_SIZE', 5000).to_i
+        if job.present?
+          if job.cron != target_job.cron && job.klass.to_s != target_job.klass
+            job.destroy
+            target_job.save
+          end
+        else
+          target_job.save
         end
       end
+
     end
   end
 end

data/lib/sidekiq/heartbeat_monitor/scheduler.rb

@@ -1,37 +1,53 @@
 module Sidekiq
   module HeartbeatMonitor
     class Scheduler
+      TIME_BETWEEN_HEARTBEATS = 60
+
       include Sidekiq::Worker
+      include Sidekiq::HeartbeatMonitor::Util
       sidekiq_options retry: 0
 
       ##
       # Checks to see if queues are backed up by 1000 or more jobs and also schedules the heartbeat job.
       def perform
-        Sidekiq::Queue.all.each do |q|
-          check_queue_size(q, Sidekiq::HeartbeatMonitor::Config.max_queue_size)
-
-          Sidekiq::HeartbeatMonitor::Worker.client_push(
-            'class' => Sidekiq::HeartbeatMonitor::Worker, 
-            'args'  => [q.name, 15], 
-            'queue' => q.name
-          )
+        Sidekiq.redis do |redis|
+          Sidekiq::Queue.all.each do |q|
+            queue_config = Sidekiq::HeartbeatMonitor.config(q)
+            check_queue_size!(q, queue_config)
+
+            key = "Sidekiq:HeartbeatMonitor:Worker-#{q.name}.enqueued_at"
+
+            last_enqueued_at = redis.get(key).to_i
+
+            if last_enqueued_at > 577997505 # Enqueued after Jan 2, 2020 when this code was written
+              time_since_enqueued = Time.now.to_i - last_enqueued_at
+              if (time_since_enqueued - TIME_BETWEEN_HEARTBEATS) > queue_config.max_heartbeat_delay
+                queue_config.send_slowed_down_alert!("⚠️ _#{q.name}_ queue is taking longer than #{format_time_str(time_since_enqueued)} to reach jobs.", q)
+              else
+                next
+              end
+            end
+
+            redis.set(key, Time.now.to_i, ex: 1.week)
+
+            Sidekiq::HeartbeatMonitor::Worker.client_push(
+              'class' => Sidekiq::HeartbeatMonitor::Worker, 
+              'args'  => [q.name], 
+              'queue' => q.name
+            )
+          end
+
         end
       end
 
-      def check_queue_size(q, max_queue_size)
+      def check_queue_size!(q, queue_config)
+        max_queue_size = queue_config.max_queue_size
+
         if q.size > max_queue_size
-          send_server_alert("⚠️ Queue #{q.name} has more than #{max_queue_size} jobs waiting to be processed. Current size is #{q.size}", q)
+         queue_config.send_backed_up_alert!("⚠️ _#{q.name}_ queue has more than #{max_queue_size} jobs waiting to be processed. Current size is #{q.size}", q)
         end
       end
 
-      ##
-      # @param msg [String] Message to post
-      # @param queue_name [Sidekiq::Queue] Queue we're concerned with
-      def send_server_alert(msg, q)
-        Sidekiq::HeartbeatMonitor::Config.send_backed_up_alert(msg, q)
-
-        true
-      end
     end
   end
 end

data/lib/sidekiq/heartbeat_monitor/util.rb

@@ -0,0 +1,20 @@
+
+module Sidekiq
+  module HeartbeatMonitor
+    module Util
+
+      def format_time_str(sec_backed_up)
+        hours = (sec_backed_up - (sec_backed_up % 3600)) / 3600
+        minutes = (sec_backed_up - (sec_backed_up % 60)) / 60
+        seconds = sec_backed_up % 60
+
+        nice_backed_up_str = "#{seconds} sec"
+        nice_backed_up_str = "#{minutes} min #{nice_backed_up_str}" if minutes > 0
+        nice_backed_up_str = "#{hours} hr #{nice_backed_up_str}" if hours > 0
+
+        nice_backed_up_str
+      end
+
+    end
+  end
+end

data/lib/sidekiq/heartbeat_monitor/version.rb

@@ -1,5 +1,5 @@
 module Sidekiq
     module HeartbeatMonitor
-        VERSION = '1.0.1.4'
+        VERSION = '1.0.2.0'
     end
 end

data/lib/sidekiq/heartbeat_monitor/worker.rb

@@ -3,43 +3,32 @@ module Sidekiq
     class Worker
 
       include Sidekiq::Worker
+      include Sidekiq::HeartbeatMonitor::Util
       sidekiq_options retry: 0
 
       ##
       # Runs every x seconds and ensures that the time between jobs is consistent and
       # @param queue_name [String] Name of the queue that this heartbeat is running on.
-      def perform(queue_name, secs_between_beats = 15)
-        current_run_at = Time.now.utc.to_i
-        last_run_at = redis.get("Sidekiq/HeartbeatMonitor/Worker/#{queue_name}.last_run_at").to_i
+      def perform(queue_name)
+        Sidekiq.redis do |redis|
+          q = Sidekiq::Queue.all.find{ |q| q.name.to_s == queue_name.to_s }
+          queue_config = Sidekiq::HeartbeatMonitor.config(q)
 
-        if last_run_at > 0
-          time_since_last_run = current_run_at - last_run_at
-          if time_since_last_run > (5.minutes + secs_between_beats)
-            sec_backed_up = time_since_last_run - secs_between_beats
+          key = "Sidekiq:HeartbeatMonitor:Worker-#{queue_name}.enqueued_at"
+          enqueued_at = redis.get(key).to_i
 
-            send_server_alert("⚠️ queue #{queue_name} took > #{sec_backed_up}s to reach job.", queue_name)
-          end
-        end
+          return if enqueued_at < 1577997505 # Enqueued before Jan 2, 2020 when this code was written
 
-        redis.set("Sidekiq/HeartbeatMonitor/Worker/#{queue_name}.last_run_at", current_run_at, ex: 1.hour)
-      end
-
-      ##
-      # @param msg [String] Message to post
-      # @param queue_name [String] Queue we're concerned with
-      def send_server_alert(msg, queue_name)
-        queue = Sidekiq::Queue.all.find{ |q| q.name.to_s == queue_name.to_s }
+          time_since_enqueued = Time.now.to_i - enqueued_at
 
-        Sidekiq::HeartbeatMonitor::Config.send_slowed_down_alert(msg, queue)
+          if time_since_enqueued > queue_config.max_heartbeat_delay
+            queue_config.send_slowed_down_alert!("⚠️ _#{queue_name}_ queue took #{format_time_str(time_since_enqueued)} to reach job.", q)
+          end
 
-        true
+          redis.del(key)
+        end
       end
 
-      private
-
-      def redis
-        @@redis ||= defined?($redis) ? $redis : Redis.new
-      end
 
     end
   end

data/sidekiq-heartbeat_monitor.gemspec

@@ -31,6 +31,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "sidekiq-cron", ">= 0.6"
   spec.add_dependency "dont_repeat_for", ">= 1"
   spec.add_dependency "slack-notifier", ">= 0.5"
+  spec.add_dependency "dotenv", ">= 1.0"
 
   spec.add_development_dependency "rails", ">= 4"
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-heartbeat_monitor
 version: !ruby/object:Gem::Version
-  version: 1.0.1.4
+  version: 1.0.2.0
 platform: ruby
 authors:
 - Jay El-Kaake
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-12-31 00:00:00.000000000 Z
+date: 2020-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: rails
   requirement: !ruby/object:Gem::Requirement
@@ -170,6 +184,7 @@ files:
 - lib/sidekiq/heartbeat_monitor.rb
 - lib/sidekiq/heartbeat_monitor/config.rb
 - lib/sidekiq/heartbeat_monitor/scheduler.rb
+- lib/sidekiq/heartbeat_monitor/util.rb
 - lib/sidekiq/heartbeat_monitor/version.rb
 - lib/sidekiq/heartbeat_monitor/worker.rb
 - sidekiq-heartbeat_monitor.gemspec