resque_stuck_queue 0.4.4 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 --- 
-SHA1: 
-  metadata.gz: b23678d316199cc108bef89177912ff5eb396d89
-  data.tar.gz: 055f4654e2f966a573fd398f2f55cbc56b21b766
 SHA512: 
-  metadata.gz: 9f4df0b59b1b092d6ee389ebea5c45a98a27a35c3e12f14a6566f07349f31d7950b9f55b582cc928303d0fbd952ad06cdf2846c9ac2e6ee5e26e8f50db42f1c0
-  data.tar.gz: 315ddbab2de9dbd42bfb803f768da71ad1cb3f34715a8beb974f5d804e9427791dd79cd626c0bf2f3fded698e29ac0141ef764ec655b79a0803b9a314196e296
+  data.tar.gz: c007b83586030670c60f58d0a5e578a058a49f54588edba34367605afac5906bea2671f589a1f03f2617329e134bc3e34ea3a6959aec95920bf2b01c63be38c1
+  metadata.gz: 86e182c1fd44835e6b899fbd21762ebeb8eeb5ba5b09d233cef6203e130468bf8cca071dc3bf9c0d6ce86c55e3b9fb77ce3816b261f36f40652cb71ff398669f
+SHA1: 
+  data.tar.gz: 17d8ce55b4fb7e5bb071eada32ccb76060ccc7e0
+  metadata.gz: a3bed0e656224a0df49636c93e8344374dd612f4

data/THOUGHTS

@@ -3,5 +3,5 @@
 rm redis locking (since it works by keys now, no need for it, recover/trigger ping pong).
 rm require resque?
 
-instead of trigggering once
-  add :warn_interval which will just continiously trigger for that interval until a recover hits.
+refactor tests to have an around(:suite) to run with resque beforehand (no startup time) and just run test_integration.rb 
+  (& compact dup tests etc)

data/lib/resque_stuck_queue/config.rb

@@ -6,7 +6,7 @@ module Resque
     HEARTBEAT_INTERVAL   = 5 * 60                   # send heartbeat job every 5 minutes
     WATCHER_INTERVAL     = 5                        # check key is udpated every 5 seconds.
 
-    TRIGGER_TIMEOUT      = 60 * 60                  # warn/trigger after an hour of lagtime.
+    TRIGGER_TIMEOUT      = 60 * 60                  # trigger after an hour of lagtime.
 
     # must be called by convention: type_handler
     TRIGGERED_HANDLER    = proc { |queue_name, lag| Resque::StuckQueue::LOGGER.info("Shit gone bad with them queues...on #{queue_name}. Lag time is #{lag}") }
@@ -24,6 +24,7 @@ module Resque
         :heartbeat_interval => "set to how often to push the 'heartbeat' job which will refresh the latest working time.\n\tExample:\n\tResque::StuckQueue.config[:heartbeat_interval] = 5.minutes",
         :watcher_interval            => "set to how often to check to see when the last time it worked was.\n\tExample:\n\tResque::StuckQueue.config[:watcher_interval] = 1.minute",
         :trigger_timeout    => "set to how much of a resque work lag you are willing to accept before being notified. note: take the :watcher_interval setting into account when setting this timeout.\n\tExample:\n\tResque::StuckQueue.config[:trigger_timeout] = 9.minutes",
+        :warn_interval      => "optional: if set, it will continiously trigger/warn in spaces of this interval after first trigger. eg, as long as lagtime keeps on being above trigger_timeout/recover hasn't occured yet.",
         :redis              => "set the Redis StuckQueue will use. Either a Redis or Redis::Namespace instance.",
         :heartbeat_key      => "optional, name of keys to keep track of the last good resque heartbeat time",
         :triggered_key      => "optional, name of keys to keep track of the last trigger time",

data/lib/resque_stuck_queue/version.rb

@@ -1,5 +1,5 @@
 module Resque
   module StuckQueue
-    VERSION = "0.4.4"
+    VERSION = "0.5.1"
   end
 end

data/lib/resque_stuck_queue.rb

@@ -90,6 +90,8 @@ module Resque
         setup_heartbeat_thread
         setup_watcher_thread
 
+        setup_warn_thread
+
         # fo-eva.
         @threads.map(&:join)
 
@@ -158,24 +160,70 @@ module Resque
 
       private
 
+      def log_starting_thread(type)
+        interval_keyname = "#{type}_interval".to_sym
+        logger.info("Starting #{type} thread with interval of #{config[interval_keyname]} seconds")
+      end
+
       def read_from_redis(keyname)
         redis.get(keyname)
       end
 
+      def setup_watcher_thread
+        @threads << Thread.new do
+          Thread.current.abort_on_exception = abort_on_exception
+          log_starting_thread(:watcher)
+          while @running
+            mutex = Redis::Mutex.new('resque_stuck_queue_lock', block: 0)
+            if mutex.lock
+              begin
+                queues.each do |queue_name|
+                  log_watcher_info(queue_name)
+                  if should_trigger?(queue_name)
+                    trigger_handler(queue_name, :triggered)
+                  elsif should_recover?(queue_name)
+                    trigger_handler(queue_name, :recovered)
+                  end
+                end
+              ensure
+                mutex.unlock
+              end
+            end
+            wait_for_it(:watcher_interval)
+          end
+        end
+      end
+
       def setup_heartbeat_thread
         @threads << Thread.new do
           Thread.current.abort_on_exception = abort_on_exception
-          logger.info("Starting heartbeat thread")
+          log_starting_thread(:heartbeat)
           while @running
             # we want to go through resque jobs, because that's what we're trying to test here:
             # ensure that jobs get executed and the time is updated!
+            wait_for_it(:heartbeat_interval)
             logger.info("Sending heartbeat jobs")
             enqueue_jobs
-            wait_for_it(:heartbeat_interval)
           end
         end
       end
 
+      def setup_warn_thread
+        if config[:warn_interval]
+          @threads << Thread.new do
+            Thread.current.abort_on_exception = abort_on_exception
+            log_starting_thread(:warn)
+            while @running
+              queues.each do |qn|
+                trigger_handler(qn, :triggered) if should_trigger?(qn, true)
+              end
+              wait_for_it(:warn_interval)
+            end
+          end
+        end
+      end
+
+
       def enqueue_jobs
         if config[:heartbeat_job]
           # FIXME config[:heartbeat_job] with mutliple queues is bad semantics
@@ -189,31 +237,6 @@ module Resque
         end
       end
 
-      def setup_watcher_thread
-        @threads << Thread.new do
-          Thread.current.abort_on_exception = abort_on_exception
-          logger.info("Starting watcher thread")
-          while @running
-            mutex = Redis::Mutex.new('resque_stuck_queue_lock', block: 0)
-            if mutex.lock
-              begin
-                queues.each do |queue_name|
-                  log_watcher_info(queue_name)
-                  if should_trigger?(queue_name)
-                    trigger_handler(queue_name, :triggered)
-                  elsif should_recover?(queue_name)
-                    trigger_handler(queue_name, :recovered)
-                  end
-                end
-              ensure
-                mutex.unlock
-              end
-            end
-            wait_for_it(:watcher_interval)
-          end
-        end
-      end
-
       def last_successful_heartbeat(queue_name)
         time_set = read_from_redis(heartbeat_key_for(queue_name))
         if time_set
@@ -255,17 +278,22 @@ module Resque
           lag_time(queue_name) < max_wait_time
       end
 
-      def should_trigger?(queue_name)
+      def should_trigger?(queue_name, force_trigger = false)
         if lag_time(queue_name) >= max_wait_time
           last_trigger = last_triggered(queue_name)
 
+          if force_trigger
+            return true
+          end
+
           if last_trigger.nil?
+            # if it hasn't been triggered before, do it
             return true
-          else
-            # if it already triggered in the past and needs to re-trigger,
-            # :recovered should have cleared last_triggered out by then
-            return false
           end
+
+          # if it already triggered in the past don't trigger again.
+          # :recovered should clearn out last_triggered so the cycle (trigger<->recover) continues
+          return false
         end
       end
 
@@ -274,8 +302,10 @@ module Resque
           sleep config[:heartbeat_interval] || HEARTBEAT_INTERVAL
         elsif type == :watcher_interval
           sleep config[:watcher_interval]   || WATCHER_INTERVAL
+        elsif type == :warn_interval
+          sleep config[:warn_interval]
         else
-          raise 'Must sleep for :watcher_interval interval or :heartbeat_interval interval!'
+          raise 'Must sleep for :watcher_interval interval or :heartbeat_interval or :warn_interval interval!'
         end
       end
 

data/test/test_helper.rb

@@ -26,7 +26,7 @@ module TestHelper
 
   def hax_kill_resque
     # ugly, FIXME how to get pid of forked forked process. run_resque pid is incorrect.
-   `ps aux |grep resque | grep -v stuck_queue |awk '{print $2}' |xargs kill`
+   `ps aux |grep -E 'resque.*(Waiting|Forked|Processing)'| grep -v grep | awk '{print $2}' |xargs kill`
    sleep 2 # wait for shutdown
   end
 

data/test/test_integration.rb

@@ -69,6 +69,67 @@ class TestIntegration < Minitest::Test
     end
   end
 
+  # warn_interval #0
+  def test_resque_does_not_enqueues_a_job_does_trigger_once_with_no_warn_interval
+  puts "#{__method__}"
+
+    with_no_resque_failures do
+      Resque::StuckQueue.config[:heartbeat_interval] = 5 # so heartbeats don't go through at all in this timeframe
+      Resque::StuckQueue.config[:trigger_timeout] = 2
+      Resque::StuckQueue.config[:watcher_interval] = 1
+      Resque::StuckQueue.config[:warn_interval] = nil
+      Resque::StuckQueue.config[:redis] = Redis.new
+      Resque::StuckQueue.config[:triggered_handler] = proc { Resque::StuckQueue.redis.incr("test_incr_warn") }
+
+      start_and_stop_loops_after(5)
+      # check handler did get called once as there is no warn_interval
+      assert_equal Resque::StuckQueue.redis.get("test_incr_warn").to_i, 1
+    end
+  end
+
+
+  # warn_interval #1
+  def test_resque_does_not_enqueues_a_job_does_trigger_with_warn_interval
+  puts "#{__method__}"
+
+    with_no_resque_failures do
+      Resque::StuckQueue.config[:heartbeat_interval] = 5 # so heartbeats don't go through at all in this timeframe
+      Resque::StuckQueue.config[:trigger_timeout] = 2
+      Resque::StuckQueue.config[:watcher_interval] = 1
+      Resque::StuckQueue.config[:warn_interval] = 1
+      Resque::StuckQueue.config[:redis] = Redis.new
+      Resque::StuckQueue.config[:triggered_handler] = proc { Resque::StuckQueue.redis.incr("test_incr_warn") }
+
+      start_and_stop_loops_after(5)
+      # check handler did get called multiple times due to warn_interval
+      assert_equal Resque::StuckQueue.redis.get("test_incr_warn").to_i, 3
+    end
+  end
+
+  # warn_interval #2
+  def test_resque_does_not_enqueues_a_job_does_trigger_with_warn_interval_stops_on_recover
+  puts "#{__method__}"
+
+    with_no_resque_failures do
+      Resque::StuckQueue.config[:heartbeat_interval] = 2 # so we trigger, and recover in this timeframe
+      Resque::StuckQueue.config[:trigger_timeout] = 2
+      Resque::StuckQueue.config[:watcher_interval] = 1
+      Resque::StuckQueue.config[:warn_interval] = 1
+      Resque::StuckQueue.config[:redis] = Redis.new
+      Resque::StuckQueue.config[:triggered_handler] = proc { Resque::StuckQueue.redis.incr("test_incr_warn") }
+
+      @recovered = false
+      Resque::StuckQueue.config[:recovered_handler] = proc { @recovered = true }
+
+      start_and_stop_loops_after(5)
+
+      assert @recovered, "resque should have picked up heartbeat job after 2 seconds"
+
+      # check handler did get called multiple times due to warn_interval but less than previous test because recover
+      assert_equal Resque::StuckQueue.redis.get("test_incr_warn").to_i, 2
+    end
+  end
+
   def test_resque_does_not_enqueues_a_job_does_trigger
     puts "#{__method__}"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: resque_stuck_queue
 version: !ruby/object:Gem::Version 
-  version: 0.4.4
+  version: 0.5.1
 platform: ruby
 authors: 
 - Shai Rosenfeld
@@ -78,7 +78,6 @@ files:
 - test/test_integration.rb
 - test/test_lagtime.rb
 - test/test_named_queues.rb
-- test/test_resque_2.rb
 - test/test_resque_stuck_queue.rb
 - test/test_set_custom_refresh_job.rb
 - test/test_ver_2.rb
@@ -113,7 +112,6 @@ test_files:
 - test/test_integration.rb
 - test/test_lagtime.rb
 - test/test_named_queues.rb
-- test/test_resque_2.rb
 - test/test_resque_stuck_queue.rb
 - test/test_set_custom_refresh_job.rb
 - test/test_ver_2.rb

data/test/test_resque_2.rb

@@ -1,45 +0,0 @@
-# run with
-# $ RESQUE_2=1 bi; RESQUE_2=1 be ruby -I. -Ilib/ test/test_resque_2.rb
-if !ENV['RESQUE_2'].nil?
-
-  require File.join(File.expand_path(File.dirname(__FILE__)), "test_helper")
-
-  class TestResque2 < Minitest::Test
-
-    include TestHelper
-
-    def setup
-      assert (Resque::VERSION.match /^2\./), "must run in 2.0"
-    Resque.redis = Redis.new
-    Resque::StuckQueue.config[:redis] = Redis.new
-      Redis.new.flushall
-    end
-
-   def test_works_with_2_point_oh_do_not_trigger_because_key_is_updated
-
-     Resque::StuckQueue.config[:redis] = Redis.new
-
-     Resque::StuckQueue.config[:watcher_interval] = 1
-     Resque::StuckQueue.config[:heartbeat_interval] = 1
-     Resque::StuckQueue.config[:abort_on_exception] = true
-     Resque::StuckQueue.config[:trigger_timeout] = 5
-     Resque::StuckQueue.config[:logger] = Logger.new($stdout)
-     Resque::StuckQueue.config[:triggered_handler] = proc { Redis.new.incr("test-incr-key") }
-     Resque::StuckQueue.config[:redis] = Redis.new
-     Resque::StuckQueue.config[:queues] = [:app]
-
-     #binding.pry
-     Resque::StuckQueue.start_in_background
-
-     @r2_pid = fork { Resque::StuckQueue.config[:redis] = Redis.new ; Resque::Worker.new("*", :graceful_term => true).work ; Process.waitall }
-     sleep 10
-
-     # triggers once
-     assert_equal Redis.new.get("test-incr-key").to_i, 0
-     hax_kill_resque
-     Resque::StuckQueue.force_stop!
-   end
-
-  end
-
-end