resque_stuck_queue 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    Zjg2N2FjNjcwNjA2MTZiNTQ1NGJjZjQwZmNiZjI5YzcyYjcxZDdmYw==
+  data.tar.gz: !binary |-
+    ZDAxNGUwYzUxYTg5NmY2N2Q0OGQ0ZTI0MzQ2MzVlNmJiNDJjY2I3OA==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    MWMxODVkOTM5NGJlMGQwZWQ5YWU1YTNjN2Q0NWM1NGUzYmExZjBmZjFkN2Yx
+    MjA5MGIzNDliY2YwYzU5NGEwNTY4NmYxMDNhZGY5MWNkYjU0ZGM1MDhjZmNk
+    ZGU4YzU1NGM2ZTc5Y2Y0OTIyYWU0NGE0OTQwNmM1ZjdmZTZhZWM=
+  data.tar.gz: !binary |-
+    M2VmNmZmY2YwMDExNzdlYzZjM2ZhMWUxZTM4NDViNjBjYjc2NmYzOWI1NDkz
+    MjRjNmZlNmM1MjA0MGJhYzAzNWUyNWIwMjVhMGVjZTFhNjM4MjkzMDA0NjQx
+    Yjg2M2VmZDU3ZWM0MWUyODFlNTI1YWVmOWQ4NjlhNzNmM2Y0NWU=

data/.gitignore

@@ -0,0 +1 @@
+*gem

data/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+gem 'resque'
+
+# TEST
+gem 'minitest'
+gem 'mocha'
+gem 'resque-mock'
+gem 'pry'
+gem 'rake'
+gem 'm'
+gem 'resque-scheduler'

data/Gemfile.lock

@@ -0,0 +1,61 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    coderay (1.1.0)
+    m (1.3.2)
+      method_source (>= 0.6.7)
+      rake (>= 0.9.2.2)
+    metaclass (0.0.1)
+    method_source (0.8.2)
+    minitest (5.2.0)
+    mocha (0.14.0)
+      metaclass (~> 0.0.1)
+    mono_logger (1.1.0)
+    multi_json (1.8.2)
+    pry (0.9.12.4)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rack (1.5.2)
+    rack-protection (1.5.1)
+      rack
+    rake (10.1.0)
+    redis (3.0.6)
+    redis-namespace (1.4.1)
+      redis (~> 3.0.4)
+    resque (1.25.1)
+      mono_logger (~> 1.0)
+      multi_json (~> 1.0)
+      redis-namespace (~> 1.2)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    resque-mock (0.1.1)
+      resque
+    resque-scheduler (2.0.1)
+      redis (>= 2.0.1)
+      resque (>= 1.20.0)
+      rufus-scheduler
+    rufus-scheduler (2.0.19)
+      tzinfo (>= 0.3.23)
+    sinatra (1.4.4)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
+    slop (3.4.7)
+    tilt (1.4.1)
+    tzinfo (0.3.38)
+    vegas (0.1.11)
+      rack (>= 1.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  m
+  minitest
+  mocha
+  pry
+  rake
+  resque
+  resque-mock
+  resque-scheduler

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Shai Rosenfeld
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,61 @@
+## Resque stuck queue
+
+Ever run into that? Sucks, eh?
+
+This should enable a way to fire some handler when jobs aren't occurring within a certain timeframe.
+
+## How it works
+
+When you call `start` you are essentially starting two threads that will continiously run until `stop` is called or until the process shuts down.
+
+One thread is responsible for pushing a 'heartbeat' job to resque which will essentially refresh a specific key in redis every time that job is processed.
+
+The other thread is a continious loop that will check redis (bypassing resque) for that key and check what the latest time the hearbeat job successfully updated that key.
+
+It will trigger a pre-defined proc (see below) if the last time the hearbeat job updated that key is older than the trigger_timeout setting (see below).
+
+## Usage
+
+Add this to wherever you're setting up resque (config/initializers or wherever).
+
+Configure it first:
+
+<pre>
+# how often to push that 'heartbeat' job to refresh the latest time it worked.
+Resque::StuckQueue.config[:heartbeat] = 5.minutes
+
+# since there is an realistic and acceptable lag for job queues, set this to how much you're
+# willing to accept between the current time and when the last hearbeat job went through.
+#
+# obviously, take the heartbeat into consideration when setting this
+Resque::StuckQueue.config[:trigger_timeout] = 10.hours
+
+# what gets triggered when resque-stuck-queue will detect the latest heartbeat is older than the trigger_timeout time set above.
+Resque::StuckQueue.config[:handler] = proc { send_email }
+
+# optional, in case you want to set your own name for the key that will be used as the last good hearbeat time
+Resque::StuckQueue.config[:global_key] = "name-the-refresh-key-as-you-please"
+
+# optional, if you want the resque-stuck-queue threads to explicitly raise, default is false
+Resque::StuckQueue.config[:abort_on_exception] = true
+</pre>
+
+Then start it:
+
+<pre>
+Resque::StuckQueue.start                # blocking
+Resque::StuckQueue.start_in_background  # sugar for Thread.new { Resque::StuckQueue.start }
+</pre>
+
+Stopping it consists of the same idea:
+
+<pre>
+Resque::StuckQueue.stop                 # this will block until the threads end their current iteration
+Resque::StuckQueue.force_stop!          # force kill those threads and let's move on
+</pre>
+
+## Tests
+
+Run the tests:
+
+`bundle; bundle exec rake`

data/Rakefile

@@ -0,0 +1,18 @@
+require 'rake/testtask'
+
+task :default => :test
+Rake::TestTask.new do |t|
+  t.pattern = "test/test_*rb"
+end
+
+require 'resque/tasks'
+
+task :'resque:setup' do
+  # https://github.com/resque/resque/issues/773
+  # have the jobs loaded in memory
+  Dir["./test/resque/*.rb"].each {|file| require file}
+end
+
+require 'resque_scheduler/tasks'
+task "resque:scheduler_setup"
+

data/THOUGHTS

@@ -0,0 +1,21 @@
+other resources:
+
+  http://vitobotta.com/resque-automatically-kill-stuck-workers-retry-failed-jobs/#sthash.oQsaNeb5.dpbs
+  http://stackoverflow.com/questions/10757758/find-out-if-a-resque-job-is-still-running-and-kill-it-if-its-stuck
+
+heartbeat?
+
+redis scenario
+  different processes (cant share state in Resque memory etc.)
+  different boxes! (redis is shared state)
+  if this is included in servers, you have clusters os unicorn/mongrel etc, 
+    ensure it works across multiple running instances of this lib?
+
+  Thing.setup_checker_thread
+  Thing.enqueue_repeating_refresh_job
+
+reproduce the error and integrate test like that
+
+## TODOS
+
+verify @config options, raise if no handler, etc.

data/lib/resque_stuck_queue/version.rb

@@ -0,0 +1,5 @@
+module Resque
+  module StuckQueue
+    VERSION = "0.0.1"
+  end
+end

data/lib/resque_stuck_queue.rb

@@ -0,0 +1,157 @@
+require "resque_stuck_queue/version"
+
+# TODO move this require into a configurable?
+require 'resque'
+
+module Resque
+  module StuckQueue
+
+    GLOBAL_KEY        = "resque-stuck-queue"
+    VERIFIED_KEY      = "resque-stuck-queue-ran-once"
+    HEARTBEAT      = 60 * 60 # check/refresh every hour
+    TRIGGER_TIMEOUT   = 5 * 60 * 60 # warn/trigger 5 hours
+    HANDLER           = proc { $stderr.puts("Shit gone bad with them queues.") }
+
+    class << self
+
+      attr_accessor :config
+
+      # # how often we refresh the key
+      # :heartbeat  = 5 * 60
+      #
+      # # this could just be :heartbeat but it's possible there's an acceptable lag/bottleneck
+      # # in the queue that we want to allow to be before we think it's bad.
+      # :trigger_timeout = 10 * 60
+      #
+      # # The global key that will be used to check the latest time
+      # :global_key  = "resque-stuck-queue"
+      #
+      # # for threads involved here. default is false
+      # :abort_on_exception 
+      #
+      # # default handler
+      # config[:handler] = proc { send_mail }
+      def config
+        @config ||= {}
+      end
+
+      def start_in_background
+        Thread.new do
+          self.start
+        end
+      end
+
+      def stop_in_background
+        Thread.new do
+          self.start
+        end
+      end
+
+      # call this after setting config. once started you should't be allowed to modify it
+      def start
+        @running = true
+        @stopped = false
+        @threads = []
+        config.freeze
+
+        mark_first_use
+
+        Thread.abort_on_exception = config[:abort_on_exception]
+
+        enqueue_repeating_refresh_job
+        setup_checker_thread
+
+        # fo-eva.
+        @threads.map(&:join)
+
+        @stopped = true
+      end
+
+      # for tests
+      def stop
+        @config = config.dup #unfreeze
+        @running = false
+
+        # wait for clean thread shutdown
+        while @stopped == false
+          sleep 1
+        end
+      end
+
+      def force_stop!
+        @threads.map(&:kill)
+      end
+
+      private
+
+      def enqueue_repeating_refresh_job
+        @threads << Thread.new do
+          while @running
+            wait_for_it
+            # we want to go through resque jobs, because that's what we're trying to test here:
+            # ensure that jobs get executed and the time is updated!
+            #
+            # TODO REDIS 2.0 compat
+            Resque.enqueue(RefreshLatestTimestamp, global_key)
+          end
+        end
+      end
+
+      def setup_checker_thread
+        @threads << Thread.new do
+          while @running
+            wait_for_it
+            if Time.now.to_i - last_time_worked > max_wait_time
+              trigger_handler
+            end
+          end
+        end
+      end
+
+      def last_time_worked
+        time_set = read_from_redis
+        if has_been_used? && time_set.nil?
+          # if the first job ran, the redis key should always be set
+          # possible cases are (1) redis data wonky (2) resque jobs don't get run
+          trigger_handler
+        end
+        (time_set ? time_set : Time.now).to_i # don't trigger again if time is nil
+      end
+
+      def trigger_handler
+        (config[:handler] || HANDLER).call
+      end
+
+      def read_from_redis
+        Resque.redis.get(global_key)
+      end
+
+      def has_been_used?
+        Resque.redis.get(VERIFIED_KEY)
+      end
+
+      def mark_first_use
+        Resque.redis.set(VERIFIED_KEY, "true")
+      end
+
+      def wait_for_it
+        sleep config[:heartbeat] || HEARTBEAT
+      end
+
+      def global_key
+        config[:global_key] || GLOBAL_KEY
+      end
+
+      def max_wait_time
+        config[:trigger_timeout] || TRIGGER_TIMEOUT
+      end
+    end
+  end
+end
+
+class RefreshLatestTimestamp
+  @queue = :app
+  def self.perform(timestamp_key)
+    Resque.redis.set(timestamp_key, Time.now.to_i)
+  end
+end

data/resque_stuck_queue.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'resque_stuck_queue/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "resque_stuck_queue"
+  spec.version       = Resque::StuckQueue::VERSION
+  spec.authors       = ["Shai Rosenfeld"]
+  spec.email         = ["srosenfeld@engineyard.com"]
+  spec.summary       = %q{fire a handler when your queues are wonky}
+  spec.description   = %q{where the wild things are. err, when resque gets stuck}
+  spec.homepage      = "https://github.com/shaiguitar/resque_stuck_queue/"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+end

data/test/resque/refresh_latest_timestamp.rb

@@ -0,0 +1,6 @@
+class RefreshLatestTimestamp
+  @queue = :app
+  def self.perform(timestamp_key)
+    Resque.redis.set(timestamp_key, Time.now.to_i)
+  end
+end

data/test/resque/set_redis_key.rb

@@ -0,0 +1,8 @@
+# fixture job
+class SetRedisKey
+  NAME = "integration_test"
+  @queue = :app
+  def self.perform
+    Resque.redis.set(NAME, "1")
+  end
+end

data/test/test_integration.rb

@@ -0,0 +1,83 @@
+require 'minitest'
+require "minitest/autorun"
+require 'pry'
+
+$:.unshift(".")
+require 'resque_stuck_queue'
+require File.join(File.expand_path(File.dirname(__FILE__)), "resque", "set_redis_key")
+require File.join(File.expand_path(File.dirname(__FILE__)), "resque", "refresh_latest_timestamp")
+
+class TestIntegration < Minitest::Test
+
+  # TODODS there's a better way to do this.
+  #
+  #
+  # run test with  VVERBOSE=1 DEBUG=1 for more output
+  #
+  #
+  # => sleeping suckc
+  # => resque sleeps 5 between checking enqueed jobs, can be configurable?
+  #
+  # cleanup processes correctly?
+  # ps aux |grep resqu |awk '{print $2}' | xargs kill
+
+  def setup
+  end
+
+  def teardown
+    Process.kill("SIGQUIT", @resque_pid)
+    Resque::StuckQueue.stop
+    Process.waitpid(@resque_pid)
+  end
+
+  def run_resque
+    pid = fork { exec("QUEUE=* bundle exec rake --trace resque:work") }
+    sleep 3 # wait for resque to boot up
+    pid
+  end
+
+  def test_resque_enqueues_a_job_does_not_trigger
+
+    puts '1'
+    Resque::StuckQueue.config[:trigger_timeout] = 100 # wait a while so we don't trigger
+    Resque::StuckQueue.config[:heartbeat] = 2
+    @triggered = false
+    Resque::StuckQueue.config[:handler] = proc { @triggered = true }
+    Thread.new { Resque::StuckQueue.start }
+
+    # job gets enqueued successfully
+    @resque_pid = run_resque
+    Resque.redis.del(SetRedisKey::NAME)
+    Resque.enqueue(SetRedisKey)
+    sleep 6 # let resque pick up the job
+    assert_equal Resque.redis.get(SetRedisKey::NAME), "1"
+
+    # check handler did not get called
+    assert_equal @triggered, false
+  end
+
+  def test_resque_does_not_enqueues_a_job_does_trigger
+
+    puts '2'
+    Resque::StuckQueue.config[:trigger_timeout] = 2 # won't allow waiting too much and will complain (eg trigger) sooner than later
+    Resque::StuckQueue.config[:heartbeat] = 1
+    @triggered = false
+    Resque::StuckQueue.config[:handler] = proc { @triggered = true }
+    Thread.new { Resque::StuckQueue.start }
+
+    # job gets enqueued successfully
+    @resque_pid = run_resque
+    Resque.redis.del(SetRedisKey::NAME)
+    Process.kill("SIGSTOP", @resque_pid) # jic, do not process jobs so we definitely trigger
+    Resque.enqueue(SetRedisKey)
+    assert_equal Resque.redis.get(SetRedisKey::NAME), nil
+
+    # check handler did get called
+    assert_equal @triggered, true
+
+    # unstick the process so we can kill it in teardown
+    Process.kill("SIGCONT", @resque_pid)
+  end
+
+
+end

data/test/test_resque_stuck_queue.rb

@@ -0,0 +1,105 @@
+require 'minitest'
+require "minitest/autorun"
+require 'mocha'
+
+require 'resque/mock'
+
+$:.unshift(".")
+require 'resque_stuck_queue'
+
+class TestResqueStuckQueue < Minitest::Test
+
+  def teardown
+    puts 'teardown'
+    Resque::StuckQueue.unstub(:has_been_used?)
+    Resque::StuckQueue.unstub(:read_from_redis)
+  end
+
+  def setup
+    puts 'setup'
+    # clean previous test runs
+    Resque.redis.flushall
+    Resque.mock!
+    Resque::StuckQueue.config[:heartbeat]   = 1 # seconds
+    Resque::StuckQueue.config[:trigger_timeout] = 2
+    Resque::StuckQueue.config[:abort_on_exception] = true
+  end
+
+  # usually the key will be set from previous runs since it will persist (redis) between deploys etc.
+  # so you shouldn't be running into this scenario (nil key) other than
+  # 0) test setup clearing out this key
+  # 1) the VERY first time you use this lib when it first gets set.
+  # 2) redis gets wiped out
+  # 3) resque jobs never get run!
+  # this has the unfortunate meaning that if no jobs are *ever* enqueued, this lib won't catch that problem.
+  # so we split the funcationaliy to raise if no key is there, unless it's the first time it's being used since being started.
+  def test_thread_does_not_trigger_when_no_key_exists_on_first_use
+    puts '1'
+
+    # lib never ran, and key is not there
+    Resque::StuckQueue.stubs(:has_been_used?).returns(nil)
+    Resque::StuckQueue.stubs(:read_from_redis).returns(nil)
+    @triggered = false
+    Resque::StuckQueue.config[:handler] = proc { @triggered = true }
+    start_and_stop_loops_after(2)
+    assert_equal false, @triggered # "handler should not be called"
+  end
+
+  def test_thread_does_trigger_when_no_key_exists_on_any_other_use
+
+    puts '2'
+    # lib already ran, but key is not there
+    Resque::StuckQueue.stubs(:has_been_used?).returns(true)
+    Resque::StuckQueue.stubs(:read_from_redis).returns(nil)
+
+    @triggered = false
+    Resque::StuckQueue.config[:handler] = proc { @triggered = true }
+    start_and_stop_loops_after(2)
+    assert_equal true, @triggered # "handler should be called"
+  end
+
+  def test_configure_global_key
+    puts '3'
+    assert_nil Resque.redis.get("it-is-configurable"), "global key should not be set"
+    Resque::StuckQueue.config[:global_key] = "it-is-configurable"
+    start_and_stop_loops_after(2)
+    refute_nil Resque.redis.get("it-is-configurable"), "global key should be set"
+  end
+
+  def test_it_sets_a_verified_key_to_indicate_first_use
+    puts '4'
+    assert_nil Resque.redis.get(Resque::StuckQueue::VERIFIED_KEY), "should be nil before lib is used"
+    start_and_stop_loops_after(2)
+    refute_nil Resque.redis.get(Resque::StuckQueue::VERIFIED_KEY), "should set verified key after used"
+  end
+
+  def test_it_does_not_trigger_handler_if_under_max_time
+    puts '5'
+    Resque::StuckQueue.stubs(:read_from_redis).returns(Time.now.to_i)
+    @triggered = false
+    Resque::StuckQueue.config[:handler] = proc { @triggered = true }
+    start_and_stop_loops_after(2)
+    assert_equal false, @triggered # "handler should not be called"
+  end
+
+  def test_it_triggers_handler_if_over_trigger_timeout
+    puts '6'
+    last_time_too_old = Time.now.to_i - Resque::StuckQueue::TRIGGER_TIMEOUT
+    Resque::StuckQueue.stubs(:read_from_redis).returns(last_time_too_old.to_s)
+    @triggered = false
+    Resque::StuckQueue.config[:handler] = proc { @triggered = true }
+    start_and_stop_loops_after(2)
+    assert_equal true, @triggered # "handler should be called"
+  end
+
+  private
+
+  def start_and_stop_loops_after(secs)
+    ops = []
+    ops << Thread.new { Resque::StuckQueue.start }
+    ops << Thread.new { sleep secs; Resque::StuckQueue.stop }
+    ops.map(&:join)
+  end
+
+end
+

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: resque_stuck_queue
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Shai Rosenfeld
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: where the wild things are. err, when resque gets stuck
+email:
+- srosenfeld@engineyard.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- THOUGHTS
+- lib/resque_stuck_queue.rb
+- lib/resque_stuck_queue/version.rb
+- resque_stuck_queue.gemspec
+- test/resque/refresh_latest_timestamp.rb
+- test/resque/set_redis_key.rb
+- test/test_integration.rb
+- test/test_resque_stuck_queue.rb
+homepage: https://github.com/shaiguitar/resque_stuck_queue/
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: fire a handler when your queues are wonky
+test_files:
+- test/resque/refresh_latest_timestamp.rb
+- test/resque/set_redis_key.rb
+- test/test_integration.rb
+- test/test_resque_stuck_queue.rb
+has_rdoc: