resque_stuck_queue 0.0.1 → 0.0.2

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    Zjg2N2FjNjcwNjA2MTZiNTQ1NGJjZjQwZmNiZjI5YzcyYjcxZDdmYw==
+    YWNkZWJjMzVlMTZlNmI4ZDNiZjk5MTk2ZjFiOTVhZTIxYjA4ZDY1Ng==
   data.tar.gz: !binary |-
-    ZDAxNGUwYzUxYTg5NmY2N2Q0OGQ0ZTI0MzQ2MzVlNmJiNDJjY2I3OA==
+    NDlmNDlmZGI3OThiMDk2NjJiZGZhZDlmY2YxNzZhOGY0ZmU1YzMxNQ==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    MWMxODVkOTM5NGJlMGQwZWQ5YWU1YTNjN2Q0NWM1NGUzYmExZjBmZjFkN2Yx
-    MjA5MGIzNDliY2YwYzU5NGEwNTY4NmYxMDNhZGY5MWNkYjU0ZGM1MDhjZmNk
-    ZGU4YzU1NGM2ZTc5Y2Y0OTIyYWU0NGE0OTQwNmM1ZjdmZTZhZWM=
+    ODgyNjJhMjNiYjNhNWRjNjBhYzhiNGUxODNmZDhiNWJiNDViODQ1ZTBlNTdk
+    MzU2ZjU3M2MyMWY0OGE5ZWIzYTBlMGI4OTEzZGFkY2RlNjkyNWU2MmI0NjYy
+    NDQ3M2IyY2E3YTAwYzUyODExZDNkY2E2ZmI3ZDVjYzAwYWVlN2Q=
   data.tar.gz: !binary |-
-    M2VmNmZmY2YwMDExNzdlYzZjM2ZhMWUxZTM4NDViNjBjYjc2NmYzOWI1NDkz
-    MjRjNmZlNmM1MjA0MGJhYzAzNWUyNWIwMjVhMGVjZTFhNjM4MjkzMDA0NjQx
-    Yjg2M2VmZDU3ZWM0MWUyODFlNTI1YWVmOWQ4NjlhNzNmM2Y0NWU=
+    NzM4ZjQyMDQzOTI2MDQyNTExMzQyNzEwNTAxM2M0NmRiYzM2YWJmMmQyYmFk
+    MTMxZWM1YmY3Nzc2ODM0N2M3MTM3NjI2ZmYxNDFjNDc5ZmUyNmVmZmIxMGQ5
+    MjkyNWNhOWZkNTNmNWVmM2ExNjg5ZGNkOGRmOTk0ZGU5NGQwYjA=

data/Gemfile

@@ -1,6 +1,7 @@
 source 'https://rubygems.org'
 
 gem 'resque'
+gem 'redis-mutex'
 
 # TEST
 gem 'minitest'

data/Gemfile.lock

@@ -21,6 +21,10 @@ GEM
       rack
     rake (10.1.0)
     redis (3.0.6)
+    redis-classy (1.2.0)
+      redis-namespace (~> 1.0)
+    redis-mutex (2.1.1)
+      redis-classy (~> 1.2)
     redis-namespace (1.4.1)
       redis (~> 3.0.4)
     resque (1.25.1)
@@ -56,6 +60,7 @@ DEPENDENCIES
   mocha
   pry
   rake
+  redis-mutex
   resque
   resque-mock
   resque-scheduler

data/README.md

@@ -31,6 +31,9 @@ Resque::StuckQueue.config[:heartbeat] = 5.minutes
 Resque::StuckQueue.config[:trigger_timeout] = 10.hours
 
 # what gets triggered when resque-stuck-queue will detect the latest heartbeat is older than the trigger_timeout time set above.
+#
+# triggering will update the key, so you'll have to wait the trigger_timeout again 
+# in order for it to trigger again even if workers are still stale.
 Resque::StuckQueue.config[:handler] = proc { send_email }
 
 # optional, in case you want to set your own name for the key that will be used as the last good hearbeat time

data/Rakefile

@@ -1,6 +1,12 @@
 require 'rake/testtask'
 
 task :default => :test
+#task :test do
+  ## forking and what not. keep containted in each own process?
+  #Dir['./test/test_*.rb'].each do |file|
+    #system("ruby -I. -I lib/ #{file}")
+  #end
+#end
 Rake::TestTask.new do |t|
   t.pattern = "test/test_*rb"
 end

data/THOUGHTS

@@ -19,3 +19,13 @@ reproduce the error and integrate test like that
 ## TODOS
 
 verify @config options, raise if no handler, etc.
+
+- use locks to only notify/trigger once in case this is used in a server-cluster type environment adn you only want the handler triggered once
+
+- resque in stuck (t state process) SIGSTOP sticks up the checker thread?
+  or if not running
+
+- ensure it only runs from the server box and not the resque box??
+  (the deploy restarts the server but not resque workers)
+
+

data/lib/resque_stuck_queue/version.rb

@@ -1,5 +1,5 @@
 module Resque
   module StuckQueue
-    VERSION = "0.0.1"
+    VERSION = "0.0.2"
   end
 end

data/lib/resque_stuck_queue.rb

@@ -1,5 +1,9 @@
 require "resque_stuck_queue/version"
 
+# TODO rm redis-mutex dep and just do the setnx locking here
+require 'redis-mutex'
+Redis::Classy.db = Resque.redis
+
 # TODO move this require into a configurable?
 require 'resque'
 
@@ -7,8 +11,7 @@ module Resque
   module StuckQueue
 
     GLOBAL_KEY        = "resque-stuck-queue"
-    VERIFIED_KEY      = "resque-stuck-queue-ran-once"
-    HEARTBEAT      = 60 * 60 # check/refresh every hour
+    HEARTBEAT         = 60 * 60 # check/refresh every hour
     TRIGGER_TIMEOUT   = 5 * 60 * 60 # warn/trigger 5 hours
     HANDLER           = proc { $stderr.puts("Shit gone bad with them queues.") }
 
@@ -54,8 +57,6 @@ module Resque
         @threads = []
         config.freeze
 
-        mark_first_use
-
         Thread.abort_on_exception = config[:abort_on_exception]
 
         enqueue_repeating_refresh_job
@@ -101,8 +102,15 @@ module Resque
         @threads << Thread.new do
           while @running
             wait_for_it
-            if Time.now.to_i - last_time_worked > max_wait_time
-              trigger_handler
+            mutex = Redis::Mutex.new('resque_stuck_queue_lock', block: 0)
+            if mutex.lock
+              begin
+                if Time.now.to_i - last_time_worked > max_wait_time
+                  trigger_handler
+                end
+              ensure
+                mutex.unlock
+              end
             end
           end
         end
@@ -110,30 +118,28 @@ module Resque
 
       def last_time_worked
         time_set = read_from_redis
-        if has_been_used? && time_set.nil?
-          # if the first job ran, the redis key should always be set
-          # possible cases are (1) redis data wonky (2) resque jobs don't get run
-          trigger_handler
-        end
-        (time_set ? time_set : Time.now).to_i # don't trigger again if time is nil
+        if time_set
+          time_set
+        else
+          manual_refresh
+         end.to_i
+      end
+
+      def manual_refresh
+         time = Time.now.to_i
+         Resque.redis.set(global_key, time)
+         time
       end
 
       def trigger_handler
         (config[:handler] || HANDLER).call
+        manual_refresh
       end
 
       def read_from_redis
         Resque.redis.get(global_key)
       end
 
-      def has_been_used?
-        Resque.redis.get(VERIFIED_KEY)
-      end
-
-      def mark_first_use
-        Resque.redis.set(VERIFIED_KEY, "true")
-      end
-
       def wait_for_it
         sleep config[:heartbeat] || HEARTBEAT
       end

data/resque_stuck_queue.gemspec

@@ -18,6 +18,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency "redis-mutex"
+
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "rake"
 end

data/test/test_collision.rb

@@ -0,0 +1,41 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), "test_helper")
+
+class TestCollision < Minitest::Test
+
+  include TestHelper
+
+  def test_two_processes_interacting
+    puts "#{__method__}"
+    # no resque should be running here so timeouts will be reached + trigger
+    Resque.redis.del("test-incr-key")
+
+    p1 = fork { Resque.redis.client.reconnect; run_resque_stuck_daemon;  }
+    p2 = fork { Resque.redis.client.reconnect; run_resque_stuck_daemon;  }
+    p3 = fork { Resque.redis.client.reconnect; run_resque_stuck_daemon;  }
+    p4 = fork { Resque.redis.client.reconnect; run_resque_stuck_daemon;  }
+
+    Thread.new {
+      sleep 5 # let test run and trigger once occur (according to time below)
+      `kill -9 #{p1}`
+      `kill -9 #{p2}`
+      `kill -9 #{p3}`
+      `kill -9 #{p4}`
+      Process.waitpid # reap
+    }
+
+    Process.waitall
+
+    assert_equal 1, Resque.redis.get("test-incr-key").to_i
+  end
+
+  private
+
+  def run_resque_stuck_daemon
+    Resque::StuckQueue.config[:heartbeat] = 1
+    Resque::StuckQueue.config[:abort_on_exception] = true
+    Resque::StuckQueue.config[:trigger_timeout] = 4
+    Resque::StuckQueue.config[:handler] = proc { Resque.redis.incr("test-incr-key") }
+    Resque::StuckQueue.start
+  end
+
+end

data/test/test_helper.rb

@@ -0,0 +1,26 @@
+require 'minitest'
+require "minitest/autorun"
+require 'pry'
+require 'mocha'
+require 'resque/mock'
+$:.unshift(".")
+require 'resque_stuck_queue'
+require File.join(File.expand_path(File.dirname(__FILE__)), "resque", "set_redis_key")
+require File.join(File.expand_path(File.dirname(__FILE__)), "resque", "refresh_latest_timestamp")
+
+module TestHelper
+
+  def run_resque
+    pid = fork { exec("QUEUE=* bundle exec rake --trace resque:work") }
+    sleep 3 # wait for resque to boot up
+    pid
+  end
+
+  def start_and_stop_loops_after(secs)
+    ops = []
+    ops << Thread.new { Resque::StuckQueue.start }
+    ops << Thread.new { sleep secs; Resque::StuckQueue.stop }
+    ops.map(&:join)
+  end
+
+end

data/test/test_integration.rb

@@ -25,7 +25,7 @@ class TestIntegration < Minitest::Test
   end
 
   def teardown
-    Process.kill("SIGQUIT", @resque_pid)
+   `kill -9 #{@resque_pid}` # CONT falls throughs sometimes? hax, rm this and SIGSTOP/SIGCONT
     Resque::StuckQueue.stop
     Process.waitpid(@resque_pid)
   end
@@ -37,8 +37,7 @@ class TestIntegration < Minitest::Test
   end
 
   def test_resque_enqueues_a_job_does_not_trigger
-
-    puts '1'
+    puts "#{__method__}"
     Resque::StuckQueue.config[:trigger_timeout] = 100 # wait a while so we don't trigger
     Resque::StuckQueue.config[:heartbeat] = 2
     @triggered = false
@@ -57,8 +56,7 @@ class TestIntegration < Minitest::Test
   end
 
   def test_resque_does_not_enqueues_a_job_does_trigger
-
-    puts '2'
+    puts "#{__method__}"
     Resque::StuckQueue.config[:trigger_timeout] = 2 # won't allow waiting too much and will complain (eg trigger) sooner than later
     Resque::StuckQueue.config[:heartbeat] = 1
     @triggered = false
@@ -71,6 +69,7 @@ class TestIntegration < Minitest::Test
     Process.kill("SIGSTOP", @resque_pid) # jic, do not process jobs so we definitely trigger
     Resque.enqueue(SetRedisKey)
     assert_equal Resque.redis.get(SetRedisKey::NAME), nil
+    sleep 2 # allow timeout to trigger
 
     # check handler did get called
     assert_equal @triggered, true

data/test/test_resque_stuck_queue.rb

@@ -1,105 +1,53 @@
-require 'minitest'
-require "minitest/autorun"
-require 'mocha'
-
-require 'resque/mock'
-
-$:.unshift(".")
-require 'resque_stuck_queue'
+require File.join(File.expand_path(File.dirname(__FILE__)), "test_helper")
 
 class TestResqueStuckQueue < Minitest::Test
 
+  include TestHelper
+
   def teardown
-    puts 'teardown'
-    Resque::StuckQueue.unstub(:has_been_used?)
+    puts "#{__method__}"
     Resque::StuckQueue.unstub(:read_from_redis)
   end
 
   def setup
-    puts 'setup'
+    puts "#{__method__}"
     # clean previous test runs
     Resque.redis.flushall
     Resque.mock!
     Resque::StuckQueue.config[:heartbeat]   = 1 # seconds
-    Resque::StuckQueue.config[:trigger_timeout] = 2
     Resque::StuckQueue.config[:abort_on_exception] = true
   end
 
-  # usually the key will be set from previous runs since it will persist (redis) between deploys etc.
-  # so you shouldn't be running into this scenario (nil key) other than
-  # 0) test setup clearing out this key
-  # 1) the VERY first time you use this lib when it first gets set.
-  # 2) redis gets wiped out
-  # 3) resque jobs never get run!
-  # this has the unfortunate meaning that if no jobs are *ever* enqueued, this lib won't catch that problem.
-  # so we split the funcationaliy to raise if no key is there, unless it's the first time it's being used since being started.
-  def test_thread_does_not_trigger_when_no_key_exists_on_first_use
-    puts '1'
-
-    # lib never ran, and key is not there
-    Resque::StuckQueue.stubs(:has_been_used?).returns(nil)
-    Resque::StuckQueue.stubs(:read_from_redis).returns(nil)
-    @triggered = false
-    Resque::StuckQueue.config[:handler] = proc { @triggered = true }
-    start_and_stop_loops_after(2)
-    assert_equal false, @triggered # "handler should not be called"
-  end
-
-  def test_thread_does_trigger_when_no_key_exists_on_any_other_use
-
-    puts '2'
-    # lib already ran, but key is not there
-    Resque::StuckQueue.stubs(:has_been_used?).returns(true)
-    Resque::StuckQueue.stubs(:read_from_redis).returns(nil)
-
-    @triggered = false
-    Resque::StuckQueue.config[:handler] = proc { @triggered = true }
-    start_and_stop_loops_after(2)
-    assert_equal true, @triggered # "handler should be called"
-  end
-
   def test_configure_global_key
-    puts '3'
+    puts "#{__method__}"
     assert_nil Resque.redis.get("it-is-configurable"), "global key should not be set"
     Resque::StuckQueue.config[:global_key] = "it-is-configurable"
     start_and_stop_loops_after(2)
     refute_nil Resque.redis.get("it-is-configurable"), "global key should be set"
   end
 
-  def test_it_sets_a_verified_key_to_indicate_first_use
-    puts '4'
-    assert_nil Resque.redis.get(Resque::StuckQueue::VERIFIED_KEY), "should be nil before lib is used"
-    start_and_stop_loops_after(2)
-    refute_nil Resque.redis.get(Resque::StuckQueue::VERIFIED_KEY), "should set verified key after used"
-  end
-
   def test_it_does_not_trigger_handler_if_under_max_time
-    puts '5'
+    puts "#{__method__}"
+    Resque::StuckQueue.config[:trigger_timeout] = 5
     Resque::StuckQueue.stubs(:read_from_redis).returns(Time.now.to_i)
+
     @triggered = false
     Resque::StuckQueue.config[:handler] = proc { @triggered = true }
-    start_and_stop_loops_after(2)
+    start_and_stop_loops_after(3)
     assert_equal false, @triggered # "handler should not be called"
   end
 
   def test_it_triggers_handler_if_over_trigger_timeout
-    puts '6'
+    puts "#{__method__}"
+    Resque::StuckQueue.config[:trigger_timeout] = 2
     last_time_too_old = Time.now.to_i - Resque::StuckQueue::TRIGGER_TIMEOUT
     Resque::StuckQueue.stubs(:read_from_redis).returns(last_time_too_old.to_s)
+
     @triggered = false
     Resque::StuckQueue.config[:handler] = proc { @triggered = true }
     start_and_stop_loops_after(2)
     assert_equal true, @triggered # "handler should be called"
   end
 
-  private
-
-  def start_and_stop_loops_after(secs)
-    ops = []
-    ops << Thread.new { Resque::StuckQueue.start }
-    ops << Thread.new { sleep secs; Resque::StuckQueue.stop }
-    ops.map(&:join)
-  end
-
 end
 

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: resque_stuck_queue
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Shai Rosenfeld
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-04 00:00:00.000000000 Z
+date: 2014-01-06 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis-mutex
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -57,6 +71,8 @@ files:
 - resque_stuck_queue.gemspec
 - test/resque/refresh_latest_timestamp.rb
 - test/resque/set_redis_key.rb
+- test/test_collision.rb
+- test/test_helper.rb
 - test/test_integration.rb
 - test/test_resque_stuck_queue.rb
 homepage: https://github.com/shaiguitar/resque_stuck_queue/
@@ -86,6 +102,8 @@ summary: fire a handler when your queues are wonky
 test_files:
 - test/resque/refresh_latest_timestamp.rb
 - test/resque/set_redis_key.rb
+- test/test_collision.rb
+- test/test_helper.rb
 - test/test_integration.rb
 - test/test_resque_stuck_queue.rb
 has_rdoc: