resque_stuck_queue 0.3.9 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 --- 
-SHA512: 
-  data.tar.gz: b49e05a7aa945e1d8e39001f5e27f59c6b05b4ecad5818418291c80db897b9ed34f6f5d56fd9514ef1c98d637300eb5db264a0ef5163cff952b81adcb7f8dae6
-  metadata.gz: 0c690deec48cc352654f9ffd3e69de75d5770eaff6d18bc96094594fa3f9100114fbee733ce37594ffab2999888260dc876da2fb699508138676be4d7c1f1269
 SHA1: 
-  data.tar.gz: 7575899cca38bf1fcfbcd5aca7cf466434625764
-  metadata.gz: 75f40917d0166d260012a436d5695fc45ea70e69
+  data.tar.gz: 4afb591b2c6fd8c3fc37a27dd92d54af07de3e2e
+  metadata.gz: 19702480a46ba718628bcb98c6f0eab1fa0c2261
+SHA512: 
+  data.tar.gz: b05278c44a84b631e5e43a1e1ba84f3d29971e075526cf16d926ed8dbb921b8399f3da4aa4a309b4f50e90c6722a5b8dbfdf1b0219c22153cc7e558e02377b6b
+  metadata.gz: 727b6ec883cf0fdec4a23a4235675309c7ea321704e204991eddfc311432e846392432799837a153d55429b8518ddd39936caca101d8541760bf80f8c7aa1d80

data/README.md

@@ -24,17 +24,76 @@ After firing the proc, it will continue to monitor the queue, but won't call the
 
 By calling the recovered proc, it will then complain again the next time the lag is found.
 
-## Configuration Options
+## Usage
+
+Run this as a daemon somewhere alongside the app/in your setup. You'll need to configure it to your needs first:
+
+Put something like this in `config/initializers/resque-stuck-queue.rb`:
+
+<pre>
+require 'resque_stuck_queue' # or require 'resque/stuck_queue'
+require 'logger'
+
+# change to decent values that make sense for you
+Resque::StuckQueue.config[:heartbeat_interval]           = 10.seconds
+Resque::StuckQueue.config[:trigger_timeout]     = 30.seconds
+
+# create a sync/unbuffered log
+logpath = Rails.root.join('log', 'resque_stuck_queue.log')
+logfile = File.open(logpath, "a")
+logfile.sync = true
+logger = Logger.new(logfile)
+logger.formatter = Logger::Formatter.new
+Resque::StuckQueue.config[:logger] = logger
+
+Resque::StuckQueue.config[:redis]  = YOUR_REDIS
+
+# which queues to monitor
+Resque::StuckQueue.config[:queues] = [:app, :custom_queue]
+
+# handler for when a resque queue is being problematic
+Resque::StuckQueue.config[:triggered_handler] = proc { |bad_queue, lagtime|
+  msg = "[BAD] AWSM #{Rails.env}'s Resque #{bad_queue} queue lagging job execution by #{lagtime} seconds."
+  send_email(msg)
+}
+
+# handler for when a resque queue recovers
+Resque::StuckQueue.config[:recovered_handler] = proc { |good_queue, lagtime|
+  msg = "[GOOD] AWSM #{Rails.env}'s Resque #{good_queue} queue lagging job execution by #{lagtime} seconds."
+  send_email(msg)
+}
+
+</pre>
 
-Configure it first via something like:
+Then create a task to run it as a daemon (similar to how the resque rake job is implemented):
 
 <pre>
-  Resque::StuckQueue.config[:triggered_handler] = proc { send_email }
+
+# put this in lib/tasks/resque_stuck_queue.rb
+
+namespace :resque do
+  desc "Start a Resque-stuck daemon"
+  # :environment dep task should load the config via the initializer
+  task :stuck_queue => :environment do
+    Resque::StuckQueue.start
+  end
+
+end
+
 </pre>
 
-Configuration settings are below. You'll most likely at the least want to tune `:triggered_handler`,`:heartbeat` and `:trigger_timeout` settings.
+then run it via god, monit or whatever:
 
 <pre>
+$ bundle exec rake --trace resque:stuck_queue # outdated god config - https://gist.github.com/shaiguitar/298935953d91faa6bd4e
+</pre>
+
+## Configuration Options
+
+Configuration settings are below. You'll most likely at the least want to tune `:triggered_handler`,`:heartbeat_interval` and `:trigger_timeout` settings.
+
+<pre>
+
 triggered_handler:
 	set to what gets triggered when resque-stuck-queue will detect the latest heartbeat is older than the trigger_timeout time setting.
 	Example:
@@ -45,15 +104,20 @@ recovered_handler:
 	Example:
 	Resque::StuckQueue.config[:recovered_handler] = proc { |queue_name, lagtime| send_email('phew, queue #{queue_name} is ok') }
 
-heartbeat:
-	set to how often to push that 'heartbeat' job to refresh the latest time it worked.
+heartbeat_interval:
+	set to how often to push the 'heartbeat' job which will refresh the latest working time.
+	Example:
+	Resque::StuckQueue.config[:heartbeat_interval] = 5.minutes
+
+watcher_interval:
+	set to how often to check to see when the last time it worked was.
 	Example:
-	Resque::StuckQueue.config[:heartbeat] = 5.minutes
+	Resque::StuckQueue.config[:watcher_interval] = 1.minute
 
 trigger_timeout:
-	set to how much of a resque work lag you are willing to accept before being notified. note: take the :heartbeat setting into account when setting this timeout.
+	set to how much of a resque work lag you are willing to accept before being notified. note: take the :watcher_interval setting into account when setting this timeout.
 	Example:
-	Resque::StuckQueue.config[:trigger_timeout] = 55.minutes
+	Resque::StuckQueue.config[:trigger_timeout] = 9.minutes
 
 redis:
 	set the Redis StuckQueue will use. Either a Redis or Redis::Namespace instance.
@@ -68,13 +132,17 @@ logger:
 	optional, pass a Logger. Default a ruby logger will be instantiated. Needs to respond to that interface.
 
 queues:
-	optional, monitor specific queues you want to send a heartbeat/monitor to. default is :app
+	optional, monitor specific queues you want to send a heartbeat/monitor to. default is [:app]
 
 abort_on_exception:
 	optional, if you want the resque-stuck-queue threads to explicitly raise, default is false
 
 heartbeat_job:
 	optional, your own custom refreshing job. if you are using something other than resque
+
+enable_signals:
+	optional, allow resque::stuck's signal_handlers which do mostly nothing at this point.
+
 </pre>
 
 To start it:
@@ -91,42 +159,6 @@ Resque::StuckQueue.stop                 # this will block until the threads end
 Resque::StuckQueue.force_stop!          # force kill those threads and let's move on
 </pre>
 
-## Deployment/Integration
-
-* Include this in the app in a config initializer of some sort.
-
-Note though, the resque-stuck threads will live alongside the app server process so you will need to explicitely handle `start` _and_ `stop`. If you're deployed in a forking-server environment and the whatever process has this does not get restarted the threads will keep on going indefinitely.
-
-* Run this as a daemon somewhere alongside the app/in your setup.
-
-Contrived example:
-
-<pre>
-
-# put this in lib/tasks/resque_stuck_queue.rb
-
-require 'resque_stuck_queue' # or require 'resque/stuck_queue'
-
-namespace :resque do
-  desc "Start a Resque-stuck daemon"
-  task :stuck_queue do
-
-    Resque::StuckQueue.config[:heartbeat] = 10.minutes
-    Resque::StuckQueue.config[:trigger_timeout] = 1.hour
-    Resque::StuckQueue.config[:triggered_handler] = proc { |queue_name| $stderr.puts("resque queue #{queue_name} wonky!") }
-
-    Resque::StuckQueue.start # blocking operation, daemon running
-  end
-end
-
-# then:
-
-$ bundle exec rake --trace resque:stuck_queue
-
-# you can run this under god for example @ https://gist.github.com/shaiguitar/298935953d91faa6bd4e
-
-</pre>
-
 ## Sidekiq/Other redis-based job queues
 
 If you have trouble with other queues you can use this lib by setting your own custom refresh job (aka, the job that refreshes your queue specific heartbeat_key). The one thing you need to take care of is ensure whatever and however you enque your own custom job, it sets the heartbeat_key to Time.now:

data/lib/resque_stuck_queue.rb

@@ -134,7 +134,7 @@ module Resque
             # ensure that jobs get executed and the time is updated!
             logger.info("Sending heartbeat jobs")
             enqueue_jobs
-            wait_for_it
+            wait_for_it(:heartbeat_interval)
           end
         end
       end
@@ -155,13 +155,13 @@ module Resque
       def setup_watcher_thread
         @threads << Thread.new do
           Thread.current.abort_on_exception = config[:abort_on_exception]
-          logger.info("Starting checker thread")
+          logger.info("Starting watcher thread")
           while @running
             mutex = Redis::Mutex.new('resque_stuck_queue_lock', block: 0)
             if mutex.lock
               begin
                 queues.each do |queue_name|
-                  log_checker_info(queue_name)
+                  log_watcher_info(queue_name)
                   if should_trigger?(queue_name)
                     trigger_handler(queue_name, :triggered)
                   elsif should_recover?(queue_name)
@@ -172,7 +172,7 @@ module Resque
                 mutex.unlock
               end
             end
-            wait_for_it
+            wait_for_it(:watcher_interval)
           end
         end
       end
@@ -248,7 +248,7 @@ module Resque
         logger.info("Starting StuckQueue with config: #{self.config.inspect}")
       end
 
-      def log_checker_info(queue_name)
+      def log_watcher_info(queue_name)
         logger.info("Lag time for #{queue_name} is #{lag_time(queue_name).inspect} seconds.")
         if triggered_ago = last_triggered(queue_name)
           logger.info("Last triggered for #{queue_name} is #{triggered_ago.inspect} seconds.")
@@ -262,8 +262,14 @@ module Resque
         redis.get(keyname)
       end
 
-      def wait_for_it
-        sleep config[:heartbeat] || HEARTBEAT_TIMEOUT
+      def wait_for_it(type)
+        if type == :heartbeat_interval
+          sleep config[:heartbeat_interval] || HEARTBEAT_INTERVAL
+        elsif type == :watcher_interval
+          sleep config[:watcher_interval]   || WATCHER_INTERVAL
+        else
+          raise 'Must sleep for :watcher_interval interval or :heartbeat_interval interval!'
+        end
       end
 
       def max_wait_time

data/lib/resque_stuck_queue/config.rb

@@ -3,31 +3,35 @@ module Resque
 
     require 'logger'
     # defaults
-    HEARTBEAT_KEY        = "resque-stuck-queue"
-    TRIGGERED_KEY        = "resque-stuck-queue-last-triggered"
-    HEARTBEAT_TIMEOUT    = 20 * 60                  # check/refresh every 20 mins.
-    TRIGGER_TIMEOUT      = 40 * 60                  # warn/trigger after an hour (with 20 min heartbeat time).
-    LOGGER               = Logger.new($stdout)
+    HEARTBEAT_INTERVAL   = 5 * 60                   # send heartbeat job every 5 minutes
+    WATCHER_INTERVAL     = 5                        # check key is udpated every 5 seconds.
+
+    TRIGGER_TIMEOUT      = 60 * 60                  # warn/trigger after an hour of lagtime.
+
     # must be called by convention: type_handler
     TRIGGERED_HANDLER    = proc { |queue_name, lag| Resque::StuckQueue::LOGGER.info("Shit gone bad with them queues...on #{queue_name}. Lag time is #{lag}") }
     RECOVERED_HANDLER    = proc { |queue_name, lag| Resque::StuckQueue::LOGGER.info("recovered queue phew #{queue_name}. Lag time is #{lag}") }
 
+    LOGGER               = Logger.new($stdout)
+    HEARTBEAT_KEY        = "resque-stuck-queue"
+    TRIGGERED_KEY        = "resque-stuck-queue-last-triggered"
+
     class Config < Hash
 
       OPTIONS_DESCRIPTIONS = {
         :triggered_handler  => "set to what gets triggered when resque-stuck-queue will detect the latest heartbeat is older than the trigger_timeout time setting.\n\tExample:\n\tResque::StuckQueue.config[:triggered_handler] = proc { |queue_name, lagtime| send_email('queue \#{queue_name} isnt working, aaah the daemons') }",
         :recovered_handler  => "set to what gets triggered when resque-stuck-queue has triggered a problem, but then detects the queue went back down to functioning well again(it wont trigger again until it has recovered).\n\tExample:\n\tResque::StuckQueue.config[:recovered_handler] = proc { |queue_name, lagtime| send_email('phew, queue \#{queue_name} is ok') }",
-        :heartbeat          => "set to how often to push that 'heartbeat' job to refresh the latest time it worked.\n\tExample:\n\tResque::StuckQueue.config[:heartbeat] = 5.minutes",
-        :trigger_timeout    => "set to how much of a resque work lag you are willing to accept before being notified. note: take the :heartbeat setting into account when setting this timeout.\n\tExample:\n\tResque::StuckQueue.config[:trigger_timeout] = 55.minutes",
+        :heartbeat_interval => "set to how often to push the 'heartbeat' job which will refresh the latest working time.\n\tExample:\n\tResque::StuckQueue.config[:heartbeat_interval] = 5.minutes",
+        :watcher_interval            => "set to how often to check to see when the last time it worked was.\n\tExample:\n\tResque::StuckQueue.config[:watcher_interval] = 1.minute",
+        :trigger_timeout    => "set to how much of a resque work lag you are willing to accept before being notified. note: take the :watcher_interval setting into account when setting this timeout.\n\tExample:\n\tResque::StuckQueue.config[:trigger_timeout] = 9.minutes",
         :redis              => "set the Redis StuckQueue will use. Either a Redis or Redis::Namespace instance.",
-
         :heartbeat_key      => "optional, name of keys to keep track of the last good resque heartbeat time",
         :triggered_key      => "optional, name of keys to keep track of the last trigger time",
         :logger             => "optional, pass a Logger. Default a ruby logger will be instantiated. Needs to respond to that interface.",
-        :queues             => "optional, monitor specific queues you want to send a heartbeat/monitor to. default is :app",
+        :queues             => "optional, monitor specific queues you want to send a heartbeat/monitor to. default is [:app]",
         :abort_on_exception => "optional, if you want the resque-stuck-queue threads to explicitly raise, default is false",
         :heartbeat_job      => "optional, your own custom refreshing job. if you are using something other than resque",
-        :enable_signals     => "optional, allow resque::stuck's signal_handlers",
+        :enable_signals     => "optional, allow resque::stuck's signal_handlers which do mostly nothing at this point.",
       }
 
       OPTIONS = OPTIONS_DESCRIPTIONS.keys

data/lib/resque_stuck_queue/version.rb

@@ -1,5 +1,5 @@
 module Resque
   module StuckQueue
-    VERSION = "0.3.9"
+    VERSION = "0.4.2"
   end
 end

data/test/test_collision.rb

@@ -6,6 +6,7 @@ class TestCollision < Minitest::Test
 
   def setup
     Resque::StuckQueue.config[:redis] = Redis.new
+    Resque::StuckQueue.config[:watcher_interval] = 1
     Resque::StuckQueue.redis.flushall
   end
 
@@ -36,7 +37,7 @@ class TestCollision < Minitest::Test
   private
 
   def run_resque_stuck_daemon
-    Resque::StuckQueue.config[:heartbeat] = 1
+    Resque::StuckQueue.config[:heartbeat_interval] = 1
     Resque::StuckQueue.config[:abort_on_exception] = true
     Resque::StuckQueue.config[:trigger_timeout] = 3
     Resque::StuckQueue.config[:triggered_handler] = proc { Resque::StuckQueue.redis.incr("test-incr-key") }

data/test/test_config.rb

@@ -5,8 +5,9 @@ class TestConfig < Minitest::Test
   include TestHelper
 
   def setup
+    Resque::StuckQueue.config[:watcher_interval] = 1
     Resque::StuckQueue.config[:trigger_timeout] = 1
-    Resque::StuckQueue.config[:heartbeat] = 1
+    Resque::StuckQueue.config[:heartbeat_interval] = 1
     Resque::StuckQueue.config[:abort_on_exception] = true
     Resque::StuckQueue.config[:redis] = Redis.new
   end
@@ -48,6 +49,7 @@ class TestConfig < Minitest::Test
     end
   end
 
+  # FIXME
   def test_can_have_signals
     puts "#{__method__}"
     begin

data/test/test_integration.rb

@@ -39,6 +39,7 @@ class TestIntegration < Minitest::Test
   def setup
     Resque::StuckQueue.config[:redis] = Redis.new
     Resque::StuckQueue.redis.flushall
+    Resque::StuckQueue.config[:watcher_interval] = 1
     Resque::StuckQueue.config[:abort_on_exception] = true
     self.class.run_resque_before_all
     self.class.tests_ran += 1
@@ -57,7 +58,7 @@ class TestIntegration < Minitest::Test
 
     with_no_resque_failures do
       Resque::StuckQueue.config[:trigger_timeout] = 10
-      Resque::StuckQueue.config[:heartbeat] = 1
+      Resque::StuckQueue.config[:heartbeat_interval] = 1
       Resque::StuckQueue.config[:redis] = Redis.new
 
       @triggered = false
@@ -73,7 +74,7 @@ class TestIntegration < Minitest::Test
 
     with_no_resque_failures do
       Resque::StuckQueue.config[:trigger_timeout] = 0
-      Resque::StuckQueue.config[:heartbeat] = 1
+      Resque::StuckQueue.config[:heartbeat_interval] = 1
       Resque::StuckQueue.config[:redis] = Redis.new
 
       @triggered = false
@@ -89,7 +90,7 @@ class TestIntegration < Minitest::Test
 
     with_no_resque_failures do
       Resque::StuckQueue.config[:trigger_timeout] = 2 # won't allow waiting too much and will complain (eg trigger) sooner than later
-      Resque::StuckQueue.config[:heartbeat] = 1
+      Resque::StuckQueue.config[:heartbeat_interval] = 1
       Resque::StuckQueue.config[:redis] = Redis.new
 
       begin

data/test/test_lagtime.rb

@@ -15,11 +15,12 @@ class TestLagTime < Minitest::Test
     Resque::StuckQueue.config[:redis] = Redis.new
     Resque::StuckQueue.redis.flushall
     Resque::StuckQueue.config[:abort_on_exception] = true
+    Resque::StuckQueue.config[:watcher_interval] = 1
   end
 
   def test_triggers_handler_with_lagtime
     Resque::StuckQueue.config[:trigger_timeout] = 2 # won't allow waiting too much and will complain (eg trigger) sooner than later
-    Resque::StuckQueue.config[:heartbeat] = 1
+    Resque::StuckQueue.config[:heartbeat_interval] = 1
     @lagtime = 0
     Resque::StuckQueue.config[:triggered_handler] = proc { |queue_name, lagtime| @lagtime = lagtime }
     start_and_stop_loops_after(5)

data/test/test_named_queues.rb

@@ -6,9 +6,10 @@ class TestNamedQueues < Minitest::Test
 
   def setup
     Resque::StuckQueue.config[:trigger_timeout] = 1
-    Resque::StuckQueue.config[:heartbeat] = 1
+    Resque::StuckQueue.config[:heartbeat_interval] = 1
     Resque::StuckQueue.config[:abort_on_exception] = true
     Resque::StuckQueue.config[:redis] = Redis.new
+    Resque::StuckQueue.config[:watcher_interval] = 1
     Resque::StuckQueue.redis.flushall
   end
 
@@ -34,7 +35,7 @@ class TestNamedQueues < Minitest::Test
   def test_resque_enqueues_a_job_with_resqueue_running_but_on_that_queue_does_trigger
     puts "#{__method__}"
     Resque::StuckQueue.config[:trigger_timeout] = 2 # won't allow waiting too much and will complain (eg trigger) sooner than later
-    Resque::StuckQueue.config[:heartbeat] = 1
+    Resque::StuckQueue.config[:heartbeat_interval] = 1
     Resque::StuckQueue.config[:queues] = [:custom_queue_name]
     @triggered = false
     Resque::StuckQueue.config[:triggered_handler] = proc { |queue_name| @triggered = queue_name }
@@ -51,7 +52,7 @@ class TestNamedQueues < Minitest::Test
   def test_resque_enqueues_a_job_correct_queue_does_not_trigger
     puts "#{__method__}"
     Resque::StuckQueue.config[:trigger_timeout] = 2 # won't allow waiting too much and will complain (eg trigger) sooner than later
-    Resque::StuckQueue.config[:heartbeat] = 1
+    Resque::StuckQueue.config[:heartbeat_interval] = 1
     Resque::StuckQueue.config[:queues] = [:custom_queue_name, :diff_one]
     assert Resque::StuckQueue.heartbeat_keys.include?("custom_queue_name:resque-stuck-queue"), 'has global keys'
     @triggered = false
@@ -69,7 +70,7 @@ class TestNamedQueues < Minitest::Test
     puts "#{__method__}"
 
     Resque::StuckQueue.config[:trigger_timeout] = 2
-    Resque::StuckQueue.config[:heartbeat] = 1
+    Resque::StuckQueue.config[:heartbeat_interval] = 1
     Resque::StuckQueue.config[:queues] = [:app]
 
     @triggered = 0

data/test/test_resque_2.rb

@@ -24,7 +24,8 @@ if !ENV['RESQUE_2'].nil?
 
      Resque::StuckQueue.config[:redis] = Redis.new
 
-     Resque::StuckQueue.config[:heartbeat] = 1
+     Resque::StuckQueue.config[:watcher_interval] = 1
+     Resque::StuckQueue.config[:heartbeat_interval] = 1
      Resque::StuckQueue.config[:abort_on_exception] = true
      Resque::StuckQueue.config[:trigger_timeout] = 5
      Resque::StuckQueue.config[:logger] = Logger.new($stdout)

data/test/test_resque_stuck_queue.rb

@@ -13,8 +13,9 @@ class TestResqueStuckQueue < Minitest::Test
     puts "#{__method__}"
     # clean previous test runs
     Resque::StuckQueue.config[:redis] = Redis.new
+    Resque::StuckQueue.config[:watcher_interval] = 1
     Resque::StuckQueue.redis.flushall
-    Resque::StuckQueue.config[:heartbeat]   = 1 # seconds
+    Resque::StuckQueue.config[:heartbeat_interval]   = 1 # seconds
     Resque::StuckQueue.config[:abort_on_exception] = true
   end
 

data/test/test_set_custom_refresh_job.rb

@@ -7,7 +7,8 @@ class TestYourOwnRefreshJob < Minitest::Test
   def setup
     Resque::StuckQueue.reset!
     Resque::StuckQueue.config[:trigger_timeout] = 1
-    Resque::StuckQueue.config[:heartbeat] = 1
+    Resque::StuckQueue.config[:heartbeat_interval] = 1
+    Resque::StuckQueue.config[:watcher_interval] = 1
     Resque::StuckQueue.config[:abort_on_exception] = true
     Resque::StuckQueue.config[:heartbeat_job] = nil
     Resque::StuckQueue.config[:redis] = Redis.new

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: resque_stuck_queue
 version: !ruby/object:Gem::Version 
-  version: 0.3.9
+  version: 0.4.2
 platform: ruby
 authors: 
 - Shai Rosenfeld
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2014-02-01 00:00:00 Z
+date: 2014-02-02 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: redis-mutex