resque_stuck_queue 0.0.9 → 0.0.10

checksums.yaml

@@ -1,7 +1,7 @@
 --- 
 SHA1: 
-  data.tar.gz: cd305cd673750663559d00b2b6412f19becc6a50
-  metadata.gz: 27e1e7879f29cb47777bfaf71dc85dac57458812
+  data.tar.gz: 984e71835c73d5f071caca892a28cb91ac6efb8f
+  metadata.gz: 0f1326d639f3e02c6710b18bb4ba16f25edd30f9
 SHA512: 
-  data.tar.gz: 2a99f6cb2b8efa736a22b02ef89245b09f58b86e0ed9189623979e8561d59101b5e88c74c6354a8ee4f26b8a66d5c9e5473a1c471ee4daa893f57136c2f3d8b6
-  metadata.gz: 08d93e4f41bfb98204c7c3cb2c3df9291d65876f11cc339559753be824dfb42f4a79f442eab1e7af0f25daf3647e058cd924172efe0eabc04bd69fb8e8756b80
+  data.tar.gz: c21a0aab9167fc5dbf9e9d95f7625165bc306df82f509db512198eefc79cf51a1801a65af658239b44d5959a52ad68406a4527bd14a4b953d4816b2f9eb59f05
+  metadata.gz: 7f5674fd8a47fd21f08ba0114aa6f8fde2fb75e0237fa8e9cce5d1462b5cb8be9dcf8427d05c9cb19499a55835d52733c82849e7b60323b06b5d5bacfd05026d

data/README.md

@@ -6,7 +6,7 @@ This is to be used to satisfy an ops problem. There have been cases resque proce
 
 ## What is it?
 
-If resque doesn't run jobs within a certain timeframe, it will trigger a pre-defined handler of your choice. You can use this to send an email, pager duty, add more resque workers, restart resque, send you a txt...whatever suits you.
+If resque doesn't run jobs in specific queues (defaults to `@queue = :app`) within a certain timeframe, it will trigger a pre-defined handler of your choice. You can use this to send an email, pager duty, add more resque workers, restart resque, send you a txt...whatever suits you.
 
 ## How it works
 
@@ -39,10 +39,11 @@ Resque::StuckQueue.config[:trigger_timeout] = 10.hours
 Resque::StuckQueue.config[:handler] = proc { send_email }
 
 # optional, in case you want to set your own name for the key that will be used as the last good hearbeat time
+# note this will be namespaced under the specific queue it's monitoring, for eg "app:name-the-refresh-key-as-you-please"
 Resque::StuckQueue.config[:global_key] = "name-the-refresh-key-as-you-please"
 
-# optional, monitor a specific redis queue
-Resque::StuckQueue.config[:queue_name] = :app
+# optional, monitor specific queues you want to send a heartbeat/monitor
+Resque::StuckQueue.config[:queues] = [:app, :high, :my_custom_queue_name]
 
 # optional, if you want the resque-stuck-queue threads to explicitly raise, default is false
 Resque::StuckQueue.config[:abort_on_exception] = true
@@ -109,7 +110,7 @@ $ bundle exec rake --trace resque:stuck_queue
 
 ## Sidekiq/Other redis-based job queues
 
-If you have trouble with other queues you can use this lib by setting your own custom refresh job (aka, the job that refreshes the global_key). The one thing you need to take care of is ensure whatever and however you enque your own custom job, it sets the global_key to Time.now:
+If you have trouble with other queues you can use this lib by setting your own custom refresh job (aka, the job that refreshes your queue specific global_key). The one thing you need to take care of is ensure whatever and however you enque your own custom job, it sets the global_key to Time.now:
 
 <pre>
 
@@ -117,7 +118,7 @@ class CustomJob
   include Sidekiq::Worker
   def perform
     # ensure you're setting the key in the redis the job queue is using
-    $redis.set(Resque::StuckQueue.global_key, Time.now.to_i)
+    $redis.set(Resque::StuckQueue.global_key_for(queue_name), Time.now.to_i)
   end
 end
 

data/lib/resque_stuck_queue/version.rb

@@ -1,5 +1,5 @@
 module Resque
   module StuckQueue
-    VERSION = "0.0.9"
+    VERSION = "0.0.10"
   end
 end

data/lib/resque_stuck_queue.rb

@@ -50,6 +50,18 @@ module Resque
         @redis ||= (config[:redis] || Resque.redis)
       end
 
+      def global_key_for(under_queue)
+        "#{under_queue}:#{config[:global_key] || GLOBAL_KEY}"
+      end
+
+      def global_keys
+        queues.map{|q| global_key_for(q) }
+      end
+
+      def queues
+        @queues ||= (config[:queues] || [:app])
+      end
+
       def start_in_background
         Thread.new do
           Thread.current.abort_on_exception = config[:abort_on_exception]
@@ -94,6 +106,7 @@ module Resque
       def reset!
         # clean state so we can stop and start in the same process.
         @config = config.dup #unfreeze
+        @queues = nil
         @running = false
         @logger = nil
       end
@@ -102,15 +115,6 @@ module Resque
         @stopped
       end
 
-      def global_key
-        # public, for use in custom heartbeat job
-        "#{named_queue}:#{config[:global_key] || GLOBAL_KEY}"
-      end
-
-      def named_queue
-        config[:named_queue] || :app
-      end
-
       private
 
       def enqueue_repeating_refresh_job
@@ -121,17 +125,20 @@ module Resque
             # we want to go through resque jobs, because that's what we're trying to test here:
             # ensure that jobs get executed and the time is updated!
             logger.info("Sending refresh job")
-            enqueue_job
+            enqueue_jobs
             wait_for_it
           end
         end
       end
 
-      def enqueue_job
+      def enqueue_jobs
         if config[:refresh_job]
+          # FIXME config[:refresh_job] with mutliple queues is bad semantics
           config[:refresh_job].call
         else
-          Resque.enqueue(RefreshLatestTimestamp, [global_key, redis.client.host, redis.client.port])
+          queues.each do |queue_name|
+            Resque.enqueue_to(queue_name, RefreshLatestTimestamp, [global_key_for(queue_name), redis.client.host, redis.client.port])
+          end
         end
       end
 
@@ -143,9 +150,11 @@ module Resque
             mutex = Redis::Mutex.new('resque_stuck_queue_lock', block: 0)
             if mutex.lock
               begin
-                if Time.now.to_i - last_time_worked > max_wait_time
-                  logger.info("Triggering handler at #{Time.now} (pid: #{Process.pid})")
-                  trigger_handler
+                queues.each do |queue_name|
+                  if Time.now.to_i - last_time_worked(queue_name) > max_wait_time
+                    logger.info("Triggering handler for #{queue_name} at #{Time.now} (pid: #{Process.pid})")
+                    trigger_handler(queue_name)
+                  end
                 end
               ensure
                 mutex.unlock
@@ -156,31 +165,31 @@ module Resque
         end
       end
 
-      def last_time_worked
-        time_set = read_from_redis
+      def last_time_worked(queue_name)
+        time_set = read_from_redis(queue_name)
         if time_set
           time_set
         else
-          manual_refresh
+          manual_refresh(queue_name)
          end.to_i
       end
 
-      def manual_refresh
+      def manual_refresh(queue_name)
          time = Time.now.to_i
-         redis.set(global_key, time)
+         redis.set(global_key_for(queue_name), time)
          time
       end
 
-      def trigger_handler
+      def trigger_handler(queue_name)
         (config[:handler] || HANDLER).call
-        manual_refresh
+        manual_refresh(queue_name)
       rescue => e
-        logger.info("handler crashed: #{e.inspect}")
+        logger.info("handler for #{queue_name} crashed: #{e.inspect}")
         force_stop!
       end
 
-      def read_from_redis
-        redis.get(global_key)
+      def read_from_redis(queue_name)
+        redis.get(global_key_for(queue_name))
       end
 
       def wait_for_it
@@ -195,7 +204,6 @@ module Resque
 end
 
 class RefreshLatestTimestamp
-  @queue = Resque::StuckQueue.named_queue
   def self.perform(args)
     timestamp_key = args[0]
     host = args[1]

data/test/test_helper.rb

@@ -11,8 +11,8 @@ require File.join(File.expand_path(File.dirname(__FILE__)), "resque", "refresh_l
 
 module TestHelper
 
-  def run_resque
-    pid = fork { exec("QUEUE=* bundle exec rake --trace resque:work") }
+  def run_resque(queue_name = "*")
+    pid = fork { exec("QUEUE=#{queue_name} bundle exec rake --trace resque:work") }
     sleep 3 # wait for resque to boot up
     pid
   end

data/test/test_integration.rb

@@ -2,13 +2,16 @@ require 'minitest'
 require "minitest/autorun"
 require 'pry'
 
+
 $:.unshift(".")
 require 'resque_stuck_queue'
 require File.join(File.expand_path(File.dirname(__FILE__)), "resque", "set_redis_key")
 require File.join(File.expand_path(File.dirname(__FILE__)), "resque", "refresh_latest_timestamp")
+require File.join(File.expand_path(File.dirname(__FILE__)), "test_helper")
 
 class TestIntegration < Minitest::Test
 
+  include TestHelper
   # TODODS there's a better way to do this.
   #
   #
@@ -30,12 +33,6 @@ class TestIntegration < Minitest::Test
     Process.waitpid(@resque_pid)
   end
 
-  def run_resque
-    pid = fork { exec("QUEUE=* bundle exec rake --trace resque:work") }
-    sleep 3 # wait for resque to boot up
-    pid
-  end
-
   def test_resque_enqueues_a_job_does_not_trigger
     puts "#{__method__}"
     Resque::StuckQueue.config[:trigger_timeout] = 100 # wait a while so we don't trigger
@@ -84,7 +81,7 @@ class TestIntegration < Minitest::Test
     Resque::StuckQueue.config[:heartbeat] = 1
 
     begin
-      Resque::StuckQueue.config[:refresh_job] = proc { Resque.enqueue(RefreshLatestTimestamp, Resque::StuckQueue.global_key) }
+      Resque::StuckQueue.config[:refresh_job] = proc { Resque.enqueue(RefreshLatestTimestamp, Resque::StuckQueue.global_key_for(:app)) }
       @triggered = false
       Resque::StuckQueue.config[:handler] = proc { @triggered = true }
       Thread.new { Resque::StuckQueue.start }
@@ -98,4 +95,5 @@ class TestIntegration < Minitest::Test
     end
   end
 
+
 end

data/test/test_named_queues.rb

@@ -0,0 +1,68 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), "test_helper")
+
+class TestNamedQueues < Minitest::Test
+
+  include TestHelper
+
+  def setup
+    Resque::StuckQueue.config[:trigger_timeout] = 1
+    Resque::StuckQueue.config[:heartbeat] = 1
+    Resque::StuckQueue.config[:abort_on_exception] = true
+    Resque.redis.flushall
+  end
+
+  def teardown
+   `kill -9 #{@resque_pid}` if @resque_pid
+    Resque::StuckQueue.stop
+    Process.waitpid(@resque_pid) if @resque_pid
+  end
+
+  def test_no_custom_queues_defaults_to_app
+    puts "#{__method__}"
+    Resque::StuckQueue.config[:queues] = nil
+    start_and_stop_loops_after(2)
+    assert Resque::StuckQueue.global_keys.include?("app:resque-stuck-queue"), 'has global keys'
+  end
+
+  def test_has_custom_queues
+    puts "#{__method__}"
+    Resque::StuckQueue.config[:queues] = [:foo,:bar]
+    start_and_stop_loops_after(2)
+    assert Resque::StuckQueue.global_keys.include?("foo:resque-stuck-queue"), 'has global keys'
+  end
+
+  def test_resque_enqueues_a_job_with_resqueue_running_but_on_that_queue_does_trigger
+    puts "#{__method__}"
+    Resque::StuckQueue.config[:trigger_timeout] = 2 # won't allow waiting too much and will complain (eg trigger) sooner than later
+    Resque::StuckQueue.config[:heartbeat] = 1
+    @triggered = false
+    Resque::StuckQueue.config[:handler] = proc { @triggered = true }
+    Resque::StuckQueue.start_in_background
+
+    # job gets enqueued successfully
+    @resque_pid = run_resque("no-such-jobs-for-this-queue")
+    sleep 2 # allow timeout to trigger
+
+    # check handler did get called
+    assert_equal @triggered, true
+  end
+
+  def test_resque_enqueues_a_job_correct_queue_does_not_trigger
+    puts "#{__method__}"
+    Resque::StuckQueue.config[:trigger_timeout] = 2 # won't allow waiting too much and will complain (eg trigger) sooner than later
+    Resque::StuckQueue.config[:heartbeat] = 1
+    Resque::StuckQueue.config[:queues] = [:custom_queue_name, :diff_one]
+    assert Resque::StuckQueue.global_keys.include?("custom_queue_name:resque-stuck-queue"), 'has global keys'
+    @triggered = false
+    Resque::StuckQueue.config[:handler] = proc { @triggered = true }
+    @resque_pid = run_resque("custom_queue_name")
+    Resque::StuckQueue.start_in_background
+    sleep 2 # allow timeout to trigger
+
+    # check handler did not get called
+    assert_equal @triggered, false
+  end
+
+end
+
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: resque_stuck_queue
 version: !ruby/object:Gem::Version 
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors: 
 - Shai Rosenfeld
@@ -66,7 +66,7 @@ files:
 - test/test_helper.rb
 - test/test_integration.rb
 - test/test_logger.rb
-- test/test_named_queue.rb
+- test/test_named_queues.rb
 - test/test_resque_2.rb
 - test/test_resque_stuck_queue.rb
 - test/test_set_custom_refresh_job.rb
@@ -100,7 +100,7 @@ test_files:
 - test/test_helper.rb
 - test/test_integration.rb
 - test/test_logger.rb
-- test/test_named_queue.rb
+- test/test_named_queues.rb
 - test/test_resque_2.rb
 - test/test_resque_stuck_queue.rb
 - test/test_set_custom_refresh_job.rb

data/test/test_named_queue.rb

@@ -1,35 +0,0 @@
-require File.join(File.expand_path(File.dirname(__FILE__)), "test_helper")
-
-class TestNamedQueue < Minitest::Test
-
-  include TestHelper
-
-  def setup
-    Resque::StuckQueue.config[:trigger_timeout] = 1
-    Resque::StuckQueue.config[:heartbeat] = 1
-    Resque::StuckQueue.config[:abort_on_exception] = true
-  end
-
-  def teardown
-    Resque::StuckQueue.reset!
-  end
-
-  def test_no_custom_named_queue
-    puts "#{__method__}"
-    Resque::StuckQueue.config[:named_queue] = nil
-    start_and_stop_loops_after(2)
-    assert_equal Resque::StuckQueue.global_key, "app:resque-stuck-queue"
-    assert_equal Resque::StuckQueue.named_queue, :app
-  end
-
-  def test_has_custom_named_queue
-    puts "#{__method__}"
-    Resque::StuckQueue.config[:named_queue] = :foo
-    start_and_stop_loops_after(2)
-    assert_equal Resque::StuckQueue.global_key, "foo:resque-stuck-queue"
-    assert_equal Resque::StuckQueue.named_queue, :foo
-  end
-
-end
-
-