resque_stuck_queue 0.4.3 → 0.4.4

checksums.yaml

@@ -1,7 +1,7 @@
 --- 
 SHA1: 
-  metadata.gz: 4e70424faa8ec9ad5fe004e176521d4563b6b585
-  data.tar.gz: 11fb88f2d886993e99fbdd65ec2ec0f4a1b62d1a
+  metadata.gz: b23678d316199cc108bef89177912ff5eb396d89
+  data.tar.gz: 055f4654e2f966a573fd398f2f55cbc56b21b766
 SHA512: 
-  metadata.gz: 9c4b847e85b9b3741690a131a883d77cb7906099a1f8fada7b5139505e7a69f2a5c66a49c4ca3a35e0eab921d5c68fe0d43567291774d811ec69cc6256f510e7
-  data.tar.gz: 62750b5b3736c284c9f10f58b37344e4750e4e0454d68a2016636d73854186b2eed89eef9eb7456866927103003c8803112aae48582a6e5050fd68f5241c4eea
+  metadata.gz: 9f4df0b59b1b092d6ee389ebea5c45a98a27a35c3e12f14a6566f07349f31d7950b9f55b582cc928303d0fbd952ad06cdf2846c9ac2e6ee5e26e8f50db42f1c0
+  data.tar.gz: 315ddbab2de9dbd42bfb803f768da71ad1cb3f34715a8beb974f5d804e9427791dd79cd626c0bf2f3fded698e29ac0141ef764ec655b79a0803b9a314196e296

data/README.md

@@ -12,6 +12,12 @@ It will also fire a proc to notify you when it's recovered.
 
 ## How it works
 
+It's a heartbeat mechanism:
+
+![meme](http://cdn.memegenerator.net/instances/500x/43575729.jpg)
+
+Ok, seriously:
+
 When you call `start` you are essentially starting two threads that will continiously run until `stop` is called or until the process shuts down.
 
 One thread is responsible for pushing a 'heartbeat' job to resque which will essentially refresh a specific key in redis every time that job is processed.
@@ -37,20 +43,20 @@ require 'logger'
 # change to decent values that make sense for you
 Resque::StuckQueue.config[:heartbeat_interval]       = 10.seconds
 Resque::StuckQueue.config[:watcher_interval]         = 1.seconds
-Resque::StuckQueue.config[:trigger_timeout]          = 30.seconds
+Resque::StuckQueue.config[:trigger_timeout]          = 30.seconds # acceptable lagtime
 
 # which queues to monitor
 Resque::StuckQueue.config[:queues]                   = [:app, :custom_queue]
 
 # handler for when a resque queue is being problematic
 Resque::StuckQueue.config[:triggered_handler]         = proc { |bad_queue, lagtime|
-  msg = "[BAD] AWSM #{Rails.env}'s Resque #{bad_queue} queue lagging job execution by #{lagtime} seconds."
+  msg = "[BAD] APPNAME #{Rails.env}'s Resque #{bad_queue} queue lagging job execution by #{lagtime} seconds."
   send_email(msg)
 }
 
 # handler for when a resque queue recovers
 Resque::StuckQueue.config[:recovered_handler]         = proc { |good_queue, lagtime|
-  msg = "[GOOD] AWSM #{Rails.env}'s Resque #{good_queue} queue lagging job execution by #{lagtime} seconds."
+  msg = "[GOOD] APPNAME #{Rails.env}'s Resque #{good_queue} queue lagging job execution by #{lagtime} seconds."
   send_email(msg)
 }
 
@@ -137,13 +143,13 @@ queues:
 	optional, monitor specific queues you want to send a heartbeat/monitor to. default is [:app]
 
 abort_on_exception:
-	optional, if you want the resque-stuck-queue threads to explicitly raise, default is false
+	optional, if you want the resque-stuck-queue threads to explicitly raise, default is true
 
 heartbeat_job:
 	optional, your own custom refreshing job. if you are using something other than resque
 
 enable_signals:
-	optional, allow resque::stuck's signal_handlers which do mostly nothing at this point.
+	optional, allow resque::stuck's signal_handlers which do mostly nothing at this point. possible future plan: log info, reopen log file, etc.
 
 </pre>
 

data/THOUGHTS

@@ -2,3 +2,6 @@
 
 rm redis locking (since it works by keys now, no need for it, recover/trigger ping pong).
 rm require resque?
+
+instead of trigggering once
+  add :warn_interval which will just continiously trigger for that interval until a recover hits.

data/lib/resque_stuck_queue.rb

@@ -131,8 +131,37 @@ module Resque
         @stopped
       end
 
+      def trigger_handler(queue_name, type)
+        raise 'Must trigger either the recovered or triggered handler!' unless (type == :recovered || type == :triggered)
+        handler_name = :"#{type}_handler"
+        logger.info("Triggering #{type} handler for #{queue_name} at #{Time.now}.")
+        (config[handler_name] || const_get(handler_name.upcase)).call(queue_name, lag_time(queue_name))
+        manual_refresh(queue_name, type)
+      rescue => e
+        logger.info("handler #{type} for #{queue_name} crashed: #{e.inspect}")
+        logger.info("\n#{e.backtrace.join("\n")}")
+        raise e
+      end
+
+      def log_starting_info
+        logger.info("Starting StuckQueue with config: #{self.config.inspect}")
+      end
+
+      def log_watcher_info(queue_name)
+        logger.info("Lag time for #{queue_name} is #{lag_time(queue_name).inspect} seconds.")
+        if triggered_ago = last_triggered(queue_name)
+          logger.info("Last triggered for #{queue_name} is #{triggered_ago.inspect} seconds.")
+        else
+          logger.info("No last trigger found for #{queue_name}.")
+        end
+      end
+
       private
 
+      def read_from_redis(keyname)
+        redis.get(keyname)
+      end
+
       def setup_heartbeat_thread
         @threads << Thread.new do
           Thread.current.abort_on_exception = abort_on_exception
@@ -240,36 +269,6 @@ module Resque
         end
       end
 
-      def trigger_handler(queue_name, type)
-        raise 'Must trigger either the recovered or triggered handler!' unless (type == :recovered || type == :triggered)
-        handler_name = :"#{type}_handler"
-        logger.info("Triggering #{type} handler for #{queue_name} at #{Time.now}.")
-        (config[handler_name] || const_get(handler_name.upcase)).call(queue_name, lag_time(queue_name))
-        manual_refresh(queue_name, type)
-      rescue => e
-        logger.info("handler #{type} for #{queue_name} crashed: #{e.inspect}")
-        logger.info("\n#{e.backtrace.join("\n")}")
-        force_stop!
-      end
-
-      def log_starting_info
-        logger.info("Starting StuckQueue with config: #{self.config.inspect}")
-      end
-
-      def log_watcher_info(queue_name)
-        logger.info("Lag time for #{queue_name} is #{lag_time(queue_name).inspect} seconds.")
-        if triggered_ago = last_triggered(queue_name)
-          logger.info("Last triggered for #{queue_name} is #{triggered_ago.inspect} seconds.")
-        else
-          logger.info("No last trigger found for #{queue_name}.")
-        end
-
-      end
-
-      def read_from_redis(keyname)
-        redis.get(keyname)
-      end
-
       def wait_for_it(type)
         if type == :heartbeat_interval
           sleep config[:heartbeat_interval] || HEARTBEAT_INTERVAL

data/lib/resque_stuck_queue/config.rb

@@ -31,7 +31,7 @@ module Resque
         :queues             => "optional, monitor specific queues you want to send a heartbeat/monitor to. default is [:app]",
         :abort_on_exception => "optional, if you want the resque-stuck-queue threads to explicitly raise, default is true",
         :heartbeat_job      => "optional, your own custom refreshing job. if you are using something other than resque",
-        :enable_signals     => "optional, allow resque::stuck's signal_handlers which do mostly nothing at this point.",
+        :enable_signals     => "optional, allow resque::stuck's signal_handlers which do mostly nothing at this point. possible future plan: log info, reopen log file, etc.",
       }
 
       OPTIONS = OPTIONS_DESCRIPTIONS.keys

data/lib/resque_stuck_queue/signals.rb

@@ -9,13 +9,16 @@ module Resque
 
           trap("SIGUSR1") do
             ENV['SIGUSR1'] = "done be had"
+            Resque::StuckQueue.logger.info("Inspecting StuckQueue config: #{Resque::StuckQueue.config.inspect}")
+            Resque::StuckQueue.queues.each do |q| Resque::StuckQueue.log_watcher_info(q) end
             Resque::StuckQueue.logger.info("¯\_(ツ)_/¯ ...")
           end
 
-          trap("SIGUSR2") do
-            require 'pry'
-            binding.pry
-          end
+          # do something meaningful
+          #trap("SIGUSR2") do
+          #  require 'pry'
+          #  binding.pry
+          #end
 
         end
       end

data/lib/resque_stuck_queue/version.rb

@@ -1,5 +1,5 @@
 module Resque
   module StuckQueue
-    VERSION = "0.4.3"
+    VERSION = "0.4.4"
   end
 end

data/test/test_helper.rb

@@ -31,10 +31,17 @@ module TestHelper
   end
 
   def start_and_stop_loops_after(secs)
+    abort_or_not = Thread.abort_on_exception
+    Thread.abort_on_exception = Resque::StuckQueue.config[:abort_on_exception]
+
     ops = []
     ops << Thread.new { Resque::StuckQueue.start }
     ops << Thread.new { sleep secs; Resque::StuckQueue.stop }
     ops.map(&:join)
+
+  ensure
+    Thread.abort_on_exception = abort_or_not
+    Resque::StuckQueue.force_stop!
   end
 
 end

data/test/test_resque_2.rb

@@ -2,20 +2,15 @@
 # $ RESQUE_2=1 bi; RESQUE_2=1 be ruby -I. -Ilib/ test/test_resque_2.rb
 if !ENV['RESQUE_2'].nil?
 
-  require 'minitest'
-  require "minitest/autorun"
-  require "pry"
-  require "logger"
-
-  $:.unshift(".")
-  require "resque_stuck_queue"
-  require File.join(File.expand_path(File.dirname(__FILE__)), "resque", "set_redis_key")
-  require File.join(File.expand_path(File.dirname(__FILE__)), "resque", "refresh_latest_timestamp")
+  require File.join(File.expand_path(File.dirname(__FILE__)), "test_helper")
 
   class TestResque2 < Minitest::Test
 
+    include TestHelper
+
     def setup
       assert (Resque::VERSION.match /^2\./), "must run in 2.0"
+    Resque.redis = Redis.new
     Resque::StuckQueue.config[:redis] = Redis.new
       Redis.new.flushall
     end
@@ -31,6 +26,7 @@ if !ENV['RESQUE_2'].nil?
      Resque::StuckQueue.config[:logger] = Logger.new($stdout)
      Resque::StuckQueue.config[:triggered_handler] = proc { Redis.new.incr("test-incr-key") }
      Resque::StuckQueue.config[:redis] = Redis.new
+     Resque::StuckQueue.config[:queues] = [:app]
 
      #binding.pry
      Resque::StuckQueue.start_in_background
@@ -38,11 +34,9 @@ if !ENV['RESQUE_2'].nil?
      @r2_pid = fork { Resque::StuckQueue.config[:redis] = Redis.new ; Resque::Worker.new("*", :graceful_term => true).work ; Process.waitall }
      sleep 10
 
-     # did not trigger, resque picked up refresh jobs
+     # triggers once
      assert_equal Redis.new.get("test-incr-key").to_i, 0
-
-     `kill #{@r2_pid}`
-     Process.waitall
+     hax_kill_resque
      Resque::StuckQueue.force_stop!
    end
 

data/test/test_resque_stuck_queue.rb

@@ -41,14 +41,17 @@ class TestResqueStuckQueue < Minitest::Test
   def test_stops_if_handler_raises
     puts "#{__method__}"
     Resque::StuckQueue.config[:trigger_timeout] = 1 # wait a short time, will trigger
+    Resque::StuckQueue.config[:abort_on_exception] = true # bubble up the raise
     last_time_too_old = Time.now.to_i - Resque::StuckQueue::TRIGGER_TIMEOUT
     Resque::StuckQueue.config[:triggered_handler] = proc { raise "handler had bad sad!" }
-    Thread.new {
-      sleep 3 # should have triggered
-      Thread.current.abort_on_exception = true
-      assert Resque::StuckQueue.stopped?, "should stop stuck_queue if handler raises."
-    }
-    start_and_stop_loops_after(4)
+    begin
+      start_and_stop_loops_after(4)
+      sleep 4
+      assert false, "should raise"
+    rescue => e
+      puts e.inspect
+      assert true, "should raise handler bad sad #{e.inspect}"
+    end
   end
 
 end

data/test/test_ver_2.rb

@@ -0,0 +1,45 @@
+# run with
+# $ RESQUE_2=1 bi; RESQUE_2=1 be ruby -I. -Ilib/ test/test_resque_2.rb
+if !ENV['RESQUE_2'].nil?
+
+  require File.join(File.expand_path(File.dirname(__FILE__)), "test_helper")
+
+  class TestResque2 < Minitest::Test
+
+    include TestHelper
+
+    def setup
+      assert (Resque::VERSION.match /^2\./), "must run in 2.0"
+    Resque.redis = Redis.new
+    Resque::StuckQueue.config[:redis] = Redis.new
+      Redis.new.flushall
+    end
+
+   def test_works_with_2_point_oh_do_not_trigger_because_key_is_updated
+
+     Resque::StuckQueue.config[:redis] = Redis.new
+
+     Resque::StuckQueue.config[:watcher_interval] = 1
+     Resque::StuckQueue.config[:heartbeat_interval] = 1
+     Resque::StuckQueue.config[:abort_on_exception] = true
+     Resque::StuckQueue.config[:trigger_timeout] = 5
+     Resque::StuckQueue.config[:logger] = Logger.new($stdout)
+     Resque::StuckQueue.config[:triggered_handler] = proc { Redis.new.incr("test-incr-key") }
+     Resque::StuckQueue.config[:redis] = Redis.new
+     Resque::StuckQueue.config[:queues] = [:app]
+
+     #binding.pry
+     Resque::StuckQueue.start_in_background
+
+     @r2_pid = fork { Resque::StuckQueue.config[:redis] = Redis.new ; Resque::Worker.new("*", :graceful_term => true).work ; Process.waitall }
+     sleep 10
+
+     # triggers once
+     assert_equal Redis.new.get("test-incr-key").to_i, 1
+     hax_kill_resque
+     Resque::StuckQueue.force_stop!
+   end
+
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: resque_stuck_queue
 version: !ruby/object:Gem::Version 
-  version: 0.4.3
+  version: 0.4.4
 platform: ruby
 authors: 
 - Shai Rosenfeld
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2014-02-04 00:00:00 Z
+date: 2014-02-12 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: redis-mutex
@@ -81,6 +81,7 @@ files:
 - test/test_resque_2.rb
 - test/test_resque_stuck_queue.rb
 - test/test_set_custom_refresh_job.rb
+- test/test_ver_2.rb
 homepage: https://github.com/shaiguitar/resque_stuck_queue/
 licenses: 
 - MIT
@@ -115,3 +116,4 @@ test_files:
 - test/test_resque_2.rb
 - test/test_resque_stuck_queue.rb
 - test/test_set_custom_refresh_job.rb
+- test/test_ver_2.rb