resque_stuck_queue 0.0.4 → 0.0.5

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MWE1YzA3NjU4NjE1NGQ3ZjQ4OGMwNzRlZDE3Nzk4ZDhlOTc2YjM4YQ==
+    MGU4NWIyMWEyMjMyMWQwYTVjYjUxYTI5YjlmZWUwMmM0ZWYxMzhlZg==
   data.tar.gz: !binary |-
-    YTBiNGVhN2U5ZjYzMzkwOWJkNDc4Y2ViZTcyOTFjOWNlMjJmZDdiNQ==
+    NzBlMGViOGVmNzMzZTU1ZjczMzNiZDU4MjU3YTViNWNmY2YwNzU3Mw==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    ZmMxMzgzYTRhM2QyNDg4YTAyYjcxMTE4Y2RmYmNhMWJjNTBlZGVmOGMxNWRk
-    MTYxZTEyN2MyODc1NTNiODljYWRlZWUxZDgzZGExZjM5NzFkZjAzYTU5Y2M2
-    YmQyMDc0ZGFjOTlhYWFlNTM4YzFmY2M3NTI2ZTliYjcyZTA2NzY=
+    Mzg5ZTUyOWVhMjQzYTkwNmQ3YmQ2NDc0MDAzZGU4NWZhNTkxYWE3ODM0N2Jk
+    NzBlYTdlOTliMTNkOGIyYzc2ZDU2MmU4ZmJlYzllNGVhZjExNzk4MWY0Mzlj
+    MGRjM2FlNmNlY2RkZWE4NGRiNTJmN2IxOTY5M2EyZDhkZjFlMWQ=
   data.tar.gz: !binary |-
-    YWY2ODg5NGEwN2Q2ZGI3ODcyMWRmZjk5OGU3NTA0ZDgyMDkxNmQ1NzIzZTM3
-    M2ExODE3NDA4NjNiMTE0ZDE5ZDdiNWEyMDE3ZjFhNDYyZTgzYTEzNTM0M2I3
-    YWEzMjc0MjFmMDFlZGNkOTk3M2YwNWY3ZjBlZjY4MWQzMzQzOTI=
+    Yjc3ZWJmN2Y3OGM3NDk0ODMzMGJjNzcxMWIzMGRkY2Q2NWQxNTdiNGU3OTQ1
+    NDU3NmRmNDMxMmRhYzk1ZmU1OTViYTJhZmY5ZjkwNGMxYTEwZjY5N2QzZmI1
+    ZThmYWMyYWE5OTgzNGRjNGJiMGZiMDczOWQ4ZmRiNmZkMDdmNWM=

data/README.md

@@ -1,8 +1,12 @@
-## Resque stuck queue
+# Resque stuck queue
 
-Ever run into that? Sucks, eh?
+## Why?
 
-This should enable a way to fire some handler when jobs aren't occurring within a certain timeframe.
+This is to be used to satisfy an ops problem. There have been cases resque processes would stop processing jobs for unknown reasons. Other times resque wouldn't be running entirely due to deploy problems architecture/human error issues. Or on a different note, resque could be highly backed up and won't process jobs because it's too busy. This enables gaining a little insight into those issues.
+
+## What is it?
+
+If resque doesn't run jobs within a certain timeframe, it will trigger a pre-defined handler of your choice. You can use this to send an email, pager duty, add more resque workers, restart resque, send you a txt...whatever suits you.
 
 ## How it works
 
@@ -25,12 +29,12 @@ Resque::StuckQueue.config[:heartbeat] = 5.minutes
 # since there is an realistic and acceptable lag for job queues, set this to how much you're
 # willing to accept between the current time and when the last hearbeat job went through.
 #
-# obviously, take the heartbeat into consideration when setting this
+# take the heartbeat into consideration when setting this (it will fire 10 hours + 5 minutes with above heartbeat).
 Resque::StuckQueue.config[:trigger_timeout] = 10.hours
 
 # what gets triggered when resque-stuck-queue will detect the latest heartbeat is older than the trigger_timeout time set above.
 #
-# triggering will update the key, so you'll have to wait the trigger_timeout again 
+# triggering will update the key, so you'll have to wait the trigger_timeout again
 # in order for it to trigger again even if workers are still stale.
 Resque::StuckQueue.config[:handler] = proc { send_email }
 
@@ -67,7 +71,33 @@ Note though, the resque-stuck threads will live alongside the app server process
 
 * Run this as a daemon somewhere alongside the app/in your setup.
 
-<!-- TODO example -->
+Contrived example:
+
+<pre>
+
+# put this in lib/tasks/resque_stuck_queue.rb
+
+require 'resque_stuck_queue'
+
+namespace :resque do
+  desc "Start a Resque-stuck daemon"
+  task :stuck_queue do
+
+    Resque::StuckQueue.config[:heartbeat] = 10.minutes
+    Resque::StuckQueue.config[:trigger_timeout] = 1.hour
+    Resque::StuckQueue.config[:handler] = proc { $stderr.puts("resque wonky!") }
+
+    Resque::StuckQueue.start # blocking operation, daemon running
+  end
+end
+
+# then:
+
+$ bundle exec rake --trace resque:stuck_queue
+
+# you can run this under god for example @ https://gist.github.com/shaiguitar/298935953d91faa6bd4e
+
+</pre>
 
 ## Tests
 

data/THOUGHTS

@@ -3,30 +3,10 @@ other resources:
   http://vitobotta.com/resque-automatically-kill-stuck-workers-retry-failed-jobs/#sthash.oQsaNeb5.dpbs
   http://stackoverflow.com/questions/10757758/find-out-if-a-resque-job-is-still-running-and-kill-it-if-its-stuck
 
-heartbeat?
-
-redis scenario
-  different processes (cant share state in Resque memory etc.)
-  different boxes! (redis is shared state)
-  if this is included in servers, you have clusters os unicorn/mongrel etc, 
-    ensure it works across multiple running instances of this lib?
-
-  Thing.setup_checker_thread
-  Thing.enqueue_repeating_refresh_job
-
-reproduce the error and integrate test like that
-
 ## TODOS
 
-verify @config options, raise if no handler, etc.
-
-- use locks to only notify/trigger once in case this is used in a server-cluster type environment adn you only want the handler triggered once
-
-- resque in stuck (t state process) SIGSTOP sticks up the checker thread?
-  or if not running
-
-- ensure it only runs from the server box and not the resque box??
-  (the deploy restarts the server but not resque workers)
-
+verifications of @config options: raise if no handler, etc.
+add daemon example to readme
 add a 'resque_stuck_queue/tasks' bit? See tres eg
 require 'resque/stuck_queue' instead?
+ensure the logging gets flushed into log file correctly? (integration with god?)

data/lib/resque/stuck_queue.rb

@@ -0,0 +1 @@
+require File.join(File.expand_path(File.dirname(__FILE__)), "..","resque_stuck_queue")

data/lib/resque_stuck_queue.rb

@@ -1,12 +1,12 @@
 require "resque_stuck_queue/version"
 
+# TODO move this require into a configurable?
+require 'resque'
+
 # TODO rm redis-mutex dep and just do the setnx locking here
 require 'redis-mutex'
 Redis::Classy.db = Resque.redis
 
-# TODO move this require into a configurable?
-require 'resque'
-
 require 'logger'
 
 module Resque
@@ -51,13 +51,6 @@ module Resque
         end
       end
 
-      def stop_in_background
-        Thread.new do
-          Thread.current.abort_on_exception = config[:abort_on_exception]
-          self.start
-        end
-      end
-
       # call this after setting config. once started you should't be allowed to modify it
       def start
         @running = true

data/lib/resque_stuck_queue/version.rb

@@ -1,5 +1,5 @@
 module Resque
   module StuckQueue
-    VERSION = "0.0.4"
+    VERSION = "0.0.5"
   end
 end

data/test/test_collision.rb

@@ -15,7 +15,7 @@ class TestCollision < Minitest::Test
     p4 = fork { Resque.redis.client.reconnect; run_resque_stuck_daemon;  }
 
     Thread.new {
-      sleep 5 # let test run and trigger once occur (according to time below)
+      sleep 4 # let test run and trigger once occur (according to time below)
       `kill -9 #{p1}`
       `kill -9 #{p2}`
       `kill -9 #{p3}`
@@ -33,7 +33,7 @@ class TestCollision < Minitest::Test
   def run_resque_stuck_daemon
     Resque::StuckQueue.config[:heartbeat] = 1
     Resque::StuckQueue.config[:abort_on_exception] = true
-    Resque::StuckQueue.config[:trigger_timeout] = 4
+    Resque::StuckQueue.config[:trigger_timeout] = 3
     Resque::StuckQueue.config[:handler] = proc { Resque.redis.incr("test-incr-key") }
     Resque::StuckQueue.start
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: resque_stuck_queue
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Shai Rosenfeld
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-06 00:00:00.000000000 Z
+date: 2014-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis-mutex
@@ -66,6 +66,7 @@ files:
 - README.md
 - Rakefile
 - THOUGHTS
+- lib/resque/stuck_queue.rb
 - lib/resque_stuck_queue.rb
 - lib/resque_stuck_queue/version.rb
 - resque_stuck_queue.gemspec