resque-scheduler 2.0.0 → 2.0.1

data/README.markdown

@@ -82,7 +82,7 @@ to know.
         # need to require your jobs.  This can be an advantage since it's
         # less code that resque-scheduler needs to know about. But in a small
         # project, it's usually easier to just include you job classes here.
-        # So, someting like this:
+        # So, something like this:
         require 'jobs'
       end
     end
@@ -99,17 +99,6 @@ any nonempty value, they will take effect.  `VERBOSE` simply dumps more output
 to stdout.  `MUTE` does the opposite and silences all output. `MUTE`
 supersedes `VERBOSE`.
 
-NOTE: You DO NOT want to run >1 instance of the scheduler.  Doing so will
-result in the same job being queued more than once.  You only need one
-instance of the scheduler running per resque instance (regardless of number
-of machines).
-
-If the scheduler process goes down for whatever reason, the delayed items
-that should have fired during the outage will fire once the scheduler process
-is started back up again (regardless of it being on a new machine).  Missed
-scheduled jobs, however, will not fire upon recovery of the scheduler process.
-
-
 
 ### Delayed jobs
 
@@ -280,6 +269,25 @@ custom job class to support the #scheduled method:
       end
     end
 
+### Redundancy and Fail-Over
+
+*>= 2.0.1 only.  Prior to 2.0.1, it is not recommended to run multiple resque-scheduler processes and will result in duplicate jobs.*
+
+You may want to have resque-scheduler running on multiple machines for 
+redudancy.  Electing a master and failover is built in and default.  Simply
+run resque-scheduler on as many machine as you want pointing to the same
+redis instance and schedule.  The scheduler processes will use redis to
+elect a master process and detect failover when the master dies.  Precautions are
+taken to prevent jobs from potentially being queued twice during failover even
+when the clocks of the scheduler machines are slightly out of sync (or load affects
+scheduled job firing time).  If you want the gory details, look at Resque::SchedulerLocking.
+
+If the scheduler process(es) goes down for whatever reason, the delayed items
+that should have fired during the outage will fire once the scheduler process
+is started back up again (regardless of it being on a new machine).  Missed
+scheduled jobs, however, will not fire upon recovery of the scheduler process.
+Think of scheduled (recurring) jobs as cron jobs - if you stop cron, it doesn't fire
+missed jobs once it starts back up.
 
 
 ### resque-web Additions

data/lib/resque/scheduler.rb

@@ -1,11 +1,13 @@
 require 'rufus/scheduler'
 require 'thwait'
+require 'resque/scheduler_locking'
 
 module Resque
 
   class Scheduler
 
     extend Resque::Helpers
+    extend Resque::SchedulerLocking
 
     class << self
 
@@ -15,7 +17,7 @@ module Resque
       # If set, produces no output
       attr_accessor :mute
 
-      # If set, will try to update the schulde in the loop
+      # If set, will try to update the schedule in the loop
       attr_accessor :dynamic
 
       # Amount of time in seconds to sleep between polls of the delayed
@@ -47,17 +49,20 @@ module Resque
 
         # Now start the scheduling part of the loop.
         loop do
-          begin
-            handle_delayed_items
-            update_schedule if dynamic
-          rescue Errno::EAGAIN, Errno::ECONNRESET => e
-            warn e.message
+          if is_master?
+            begin
+              handle_delayed_items
+              update_schedule if dynamic
+            rescue Errno::EAGAIN, Errno::ECONNRESET => e
+              warn e.message
+            end
           end
           poll_sleep
         end
 
         # never gets here.
       end
+     
 
       # For all signals, set the shutdown flag and wait for current
       # poll/enqueing to finish (should be almost istant).  In the
@@ -133,8 +138,10 @@ module Resque
             if !config[interval_type].nil? && config[interval_type].length > 0
               args = optionizate_interval_value(config[interval_type])
               @@scheduled_jobs[name] = rufus_scheduler.send(interval_type, *args) do
-                log! "queueing #{config['class']} (#{name})"
-                handle_errors { enqueue_from_config(config) }
+                if is_master?
+                  log! "queueing #{config['class']} (#{name})"
+                  handle_errors { enqueue_from_config(config) }
+                end
               end
               interval_defined = true
               break
@@ -169,7 +176,8 @@ module Resque
         item = nil
         begin
           handle_shutdown do
-            if item = Resque.next_item_for_timestamp(timestamp)
+            # Continually check that it is still the master
+            if is_master? && item = Resque.next_item_for_timestamp(timestamp)
               log "queuing #{item['class']} [delayed]"
               handle_errors { enqueue_from_config(item) }
             end
@@ -218,7 +226,14 @@ module Resque
           # one app that schedules for another
           if Class === klass
             ResqueScheduler::Plugin.run_before_delayed_enqueue_hooks(klass, *params)
-            Resque.enqueue_to(queue, klass, *params)
+
+            # If the class is a custom job class, call self#scheduled on it. This allows you to do things like
+            # Resque.enqueue_at(timestamp, CustomJobClass). Otherwise, pass off to Resque.
+            if klass.respond_to?(:scheduled)
+              klass.scheduled(queue, klass_name, *params)
+            else
+              Resque.enqueue_to(queue, klass, *params)
+            end
           else
             # This will not run the before_hooks in rescue, but will at least
             # queue the job.
@@ -281,7 +296,10 @@ module Resque
       # Sets the shutdown flag, exits if sleeping
       def shutdown
         @shutdown = true
-        exit if @sleeping
+        if @sleeping
+          release_master_lock!
+          exit
+        end
       end
 
       def log!(msg)

data/lib/resque/scheduler/lock.rb

@@ -0,0 +1,3 @@
+%w[base basic resilient].each do |file|
+  require "resque/scheduler/lock/#{file}"
+end

data/lib/resque/scheduler/lock/base.rb

@@ -0,0 +1,52 @@
+module Resque
+  class Scheduler
+    module Lock
+      class Base
+        attr_reader :key
+        attr_accessor :timeout
+
+        def initialize(key, options = {})
+          @key = key
+
+          # 3 minute default timeout
+          @timeout = options[:timeout] || 60 * 3
+        end
+
+        # Attempts to acquire the lock. Returns true if successfully acquired.
+        def acquire!
+          raise NotImplementedError
+        end
+
+        def value
+          @value ||= [hostname, process_id].join(':')
+        end
+
+        # Returns true if you currently hold the lock.
+        def locked?
+          raise NotImplementedError
+        end
+
+        # Releases the lock.
+        def release!
+          Resque.redis.del(key) == 1
+        end
+
+      private
+
+        # Extends the lock by `timeout` seconds.
+        def extend_lock!
+          Resque.redis.expire(key, timeout)
+        end
+
+        def hostname
+          local_hostname = Socket.gethostname
+          Socket.gethostbyname(local_hostname).first rescue local_hostname
+        end
+
+        def process_id
+          Process.pid
+        end
+      end
+    end
+  end
+end

data/lib/resque/scheduler/lock/basic.rb

@@ -0,0 +1,28 @@
+require 'resque/scheduler/lock/base'
+
+module Resque
+  class Scheduler
+    module Lock
+      class Basic < Base
+        def acquire!
+          if Resque.redis.setnx(key, value)
+            extend_lock!
+            true
+          end
+        end
+
+        def locked?
+          if Resque.redis.get(key) == value
+            extend_lock!
+
+            if Resque.redis.get(key) == value
+              return true
+            end
+          end
+
+          false
+        end
+      end
+    end
+  end
+end

data/lib/resque/scheduler/lock/resilient.rb

@@ -0,0 +1,69 @@
+require 'resque/scheduler/lock/base'
+
+module Resque
+  class Scheduler
+    module Lock
+      class Resilient < Base
+        def acquire!
+          Resque.redis.evalsha(
+            acquire_sha,
+            :keys => [key],
+            :argv => [value]
+          ).to_i == 1
+        end
+
+        def locked?
+          Resque.redis.evalsha(
+            locked_sha,
+            :keys => [key],
+            :argv => [value]
+          ).to_i == 1
+        end
+
+      private
+
+        def locked_sha(refresh = false)
+          @locked_sha = nil if refresh
+
+          @locked_sha ||= begin
+            Resque.redis.script(
+              :load,
+              <<-EOF
+if redis.call('GET', KEYS[1]) == ARGV[1]
+then
+  redis.call('EXPIRE', KEYS[1], #{timeout})
+
+  if redis.call('GET', KEYS[1]) == ARGV[1]
+  then
+    return 1
+  end
+end
+
+return 0
+EOF
+            )
+          end
+        end
+
+        def acquire_sha(refresh = false)
+          @acquire_sha = nil if refresh
+
+          @acquire_sha ||= begin
+            Resque.redis.script(
+              :load,
+              <<-EOF
+if redis.call('SETNX', KEYS[1], ARGV[1]) == 1
+then
+  redis.call('EXPIRE', KEYS[1], #{timeout})
+  return 1
+else
+  return 0
+end
+EOF
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/resque/scheduler_locking.rb

@@ -0,0 +1,90 @@
+
+# ### Locking the scheduler process
+#
+# There are two places in resque-scheduler that need to be synchonized
+# in order to be able to run redundant scheduler processes while ensuring jobs don't
+# get queued multiple times when the master process changes.
+#
+# 1) Processing the delayed queues (jobs that are created from enqueue_at/enqueue_in, etc)
+# 2) Processing the scheduled (cron-like) jobs from rufus-scheduler
+#
+# Protecting the delayed queues (#1) is relatively easy.  A simple SETNX in
+# redis would suffice.  However, protecting the scheduled jobs is trickier
+# because the clocks on machines could be slightly off or actual firing times
+# could vary slightly due to load.  If scheduler A's clock is slightly ahead
+# of scheduler B's clock (since they are on different machines), when
+# scheduler A dies, we need to ensure that scheduler B doesn't queue jobs
+# that A already queued before it's death. (This all assumes that it is
+# better to miss a few scheduled jobs than it is to run them multiple times
+# for the same iteration.)
+#
+# To avoid queuing multiple jobs in the case of master fail-over, the master
+# should remain the master as long as it can rather than a simple SETNX which
+# would result in the master roll being passed around frequently.
+#
+# Locking Scheme:
+# Each resque-scheduler process attempts to get the master lock via SETNX.
+# Once obtained, it sets the expiration for 3 minutes (configurable).  The
+# master process continually updates the timeout on the lock key to be 3
+# minutes in the future in it's loop(s) (see `run`) and when jobs come out of
+# rufus-scheduler (see `load_schedule_job`).  That ensures that a minimum of
+# 3 minutes must pass since the last queuing operation before a new master is
+# chosen.  If, for whatever reason, the master fails to update the expiration
+# for 3 minutes, the key expires and the lock is up for grabs.  If
+# miraculously the original master comes back to life, it will realize it is
+# no longer the master and stop processing jobs.
+#
+# The clocks on the scheduler machines can then be up to 3 minutes off from
+# each other without the risk of queueing the same scheduled job twice during
+# a master change.  The catch is, in the event of a master change, no
+# scheduled jobs will be queued during those 3 minutes.  So, there is a trade
+# off: the higher the timeout, the less likely scheduled jobs will be fired
+# twice but greater chances of missing scheduled jobs.  The lower the timeout,
+# less likely jobs will be missed, greater the chances of jobs firing twice.  If
+# you don't care about jobs firing twice or are certain your machines' clocks
+# are well in sync, a lower timeout is preferable.  One thing to keep in mind:
+# this only effects *scheduled* jobs - delayed jobs will never be lost or
+# skipped since eventually a master will come online and it will process
+# everything that is ready (no matter how old it is).  Scheduled jobs work
+# like cron - if you stop cron, no jobs fire while it's stopped and it doesn't
+# fire jobs that were missed when it starts up again.
+
+require 'resque/scheduler/lock'
+
+module Resque
+  module SchedulerLocking
+    def master_lock
+      @master_lock ||= build_master_lock
+    end
+
+    def supports_lua?
+      redis_master_version >= 2.5
+    end
+
+    def is_master?
+      master_lock.acquire! || master_lock.locked?
+    end
+
+    def release_master_lock!
+      master_lock.release!
+    end
+
+  private
+
+    def build_master_lock
+      if supports_lua?
+        Resque::Scheduler::Lock::Resilient.new(master_lock_key)
+      else
+        Resque::Scheduler::Lock::Basic.new(master_lock_key)
+      end
+    end
+
+    def master_lock_key
+      :resque_scheduler_master_lock
+    end
+
+    def redis_master_version
+      Resque.redis.info['redis_version'].to_f
+    end
+  end
+end

data/lib/resque_scheduler.rb

@@ -123,7 +123,13 @@ module ResqueScheduler
 
     if Resque.inline?
       # Just create the job and let resque perform it right away with inline.
-      Resque::Job.create(queue, klass, *args)
+      # If the class is a custom job class, call self#scheduled on it. This allows you to do things like
+      # Resque.enqueue_at(timestamp, CustomJobClass, :opt1 => val1). Otherwise, pass off to Resque.
+      if klass.respond_to?(:scheduled)
+        klass.scheduled(queue, klass.to_s(), *args)
+      else
+        Resque::Job.create(queue, klass, *args)
+      end
     else
       delayed_push(timestamp, job_to_hash_with_queue(queue, klass, args))
     end
@@ -216,12 +222,13 @@ module ResqueScheduler
   # Given an encoded item, remove it from the delayed_queue
   #
   # This method is potentially very expensive since it needs to scan
-  # through the delayed queue for every timestamp.
+  # through the delayed queue for every timestamp, but at least it
+  # doesn't kill Redis by calling redis.keys.
   def remove_delayed(klass, *args)
     destroyed = 0
     search = encode(job_to_hash(klass, args))
-    Array(redis.keys("delayed:*")).each do |key|
-      destroyed += redis.lrem key, 0, search
+    Array(redis.zrange(:delayed_queue_schedule, 0, -1)).each do |timestamp|
+      destroyed += redis.lrem "delayed:#{timestamp}", 0, search
     end
     destroyed
   end
@@ -258,6 +265,9 @@ module ResqueScheduler
 
     def clean_up_timestamp(key, timestamp)
       # If the list is empty, remove it.
+
+      # Use a watch here to ensure nobody adds jobs to this delayed
+      # queue while we're removing it.
       redis.watch key
       if 0 == redis.llen(key).to_i
         redis.multi do
@@ -268,6 +278,7 @@ module ResqueScheduler
         redis.unwatch
       end
     end
+    
     def validate_job!(klass)
       if klass.to_s.empty?
         raise Resque::NoClassError.new("Jobs must be given a class.")

data/lib/resque_scheduler/version.rb

@@ -1,3 +1,3 @@
 module ResqueScheduler
-  VERSION = '2.0.0'
+  VERSION = '2.0.1'
 end

data/test/delayed_queue_test.rb

@@ -184,6 +184,25 @@ context "DelayedQueue" do
     Resque::Scheduler.handle_delayed_items(t)
   end
 
+  test "calls klass#scheduled when enqueuing jobs if it exists" do
+    t = Time.now - 60
+    Resque.enqueue_at(t, FakeCustomJobClassEnqueueAt, :foo => "bar")
+    FakeCustomJobClassEnqueueAt.expects(:scheduled).once.with('test', FakeCustomJobClassEnqueueAt.to_s, {"foo" => "bar"})
+    Resque::Scheduler.handle_delayed_items
+  end
+
+  test "when Resque.inline = true, calls klass#scheduled when enqueuing jobs if it exists" do
+    old_val = Resque.inline
+    begin
+      Resque.inline = true
+      t = Time.now - 60
+      FakeCustomJobClassEnqueueAt.expects(:scheduled).once.with(:test, FakeCustomJobClassEnqueueAt.to_s, {:foo => "bar"})
+      Resque.enqueue_at(t, FakeCustomJobClassEnqueueAt, :foo => "bar")
+    ensure
+      Resque.inline = old_val
+    end
+  end
+
   test "enqueue_delayed_items_for_timestamp creates jobs and empties the delayed queue" do
     t = Time.now + 60
 

data/test/redis-test.conf

@@ -106,10 +106,3 @@ databases 16
 # errors for write operations, and this may even lead to DB inconsistency.
 
 # maxmemory <bytes>
-
-############################### ADVANCED CONFIG ###############################
-
-# Glue small output buffers together in order to send small replies in a
-# single TCP packet. Uses a bit more CPU but most of the times it is a win
-# in terms of number of queries per second. Use 'yes' if unsure.
-glueoutputbuf yes

data/test/scheduler_locking_test.rb

@@ -0,0 +1,180 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+module LockTestHelper
+  def lock_is_not_held(lock)
+    Resque.redis.set(lock.key, 'anothermachine:1234')
+  end
+end
+
+context 'Resque::SchedulerLocking' do
+  setup do
+    @subject = Class.new { extend Resque::SchedulerLocking }
+  end
+
+  teardown do
+    Resque.redis.del(@subject.master_lock.key)
+  end
+
+  test 'it should use the basic lock mechanism for <= Redis 2.4' do
+    Resque.redis.stubs(:info).returns('redis_version' => '2.4.16')
+
+    assert_equal @subject.master_lock.class, Resque::Scheduler::Lock::Basic
+  end
+
+  test 'it should use the resilient lock mechanism for > Redis 2.4' do
+    Resque.redis.stubs(:info).returns('redis_version' => '2.5.12')
+
+    assert_equal @subject.master_lock.class, Resque::Scheduler::Lock::Resilient
+  end
+
+  test 'it should be the master if the lock is held' do
+    @subject.master_lock.acquire!
+    assert @subject.is_master?, 'should be master'
+  end
+
+  test 'it should not be the master if the lock is held by someone else' do
+    Resque.redis.set(@subject.master_lock.key, 'somethingelse:1234')
+    assert !@subject.is_master?, 'should not be master'
+  end
+
+  test "release_master_lock should delegate to master_lock" do
+    @subject.master_lock.expects(:release!)
+    @subject.release_master_lock!
+  end
+end
+
+context 'Resque::Scheduler::Lock::Base' do
+  setup do
+    @lock = Resque::Scheduler::Lock::Base.new('test_lock_key')
+  end
+
+  test '#acquire! should be not implemented' do
+    assert_raise(NotImplementedError) do
+      @lock.acquire!
+    end
+  end
+
+  test '#locked? should be not implemented' do
+    assert_raise(NotImplementedError) do
+      @lock.locked?
+    end
+  end
+end
+
+context 'Resque::Scheduler::Lock::Basic' do
+  include LockTestHelper
+
+  setup do
+    @lock = Resque::Scheduler::Lock::Basic.new('test_lock_key')
+  end
+
+  teardown do
+    @lock.release!
+  end
+
+  test 'you should not have the lock if someone else holds it' do
+    lock_is_not_held(@lock)
+
+    assert !@lock.locked?
+  end
+
+  test 'you should not be able to acquire the lock if someone else holds it' do
+    lock_is_not_held(@lock)
+
+    assert !@lock.acquire!
+  end
+
+  test "the lock should receive a TTL on acquiring" do
+    @lock.acquire!
+
+    assert Resque.redis.ttl(@lock.key) > 0, "lock should expire"
+  end
+
+  test 'releasing should release the master lock' do
+    assert @lock.acquire!, 'should have acquired the master lock'
+    assert @lock.locked?, 'should be locked'
+
+    @lock.release!
+
+    assert !@lock.locked?, 'should not be locked'
+  end
+
+  test 'checking the lock should increase the TTL if we hold it' do
+    @lock.acquire!
+    Resque.redis.setex(@lock.key, 10, @lock.value)
+
+    @lock.locked?
+
+    assert Resque.redis.ttl(@lock.key) > 10, "TTL should have been updated"
+  end
+
+  test 'checking the lock should not increase the TTL if we do not hold it' do
+    Resque.redis.setex(@lock.key, 10, @lock.value)
+    lock_is_not_held(@lock)
+
+    @lock.locked?
+
+    assert Resque.redis.ttl(@lock.key) <= 10, "TTL should not have been updated"
+  end
+end
+
+context 'Resque::Scheduler::Lock::Resilient' do
+  include LockTestHelper
+
+  if !Resque::Scheduler.supports_lua?
+    puts "*** Skipping Resque::Scheduler::Lock::Resilient tests, as they require Redis >= 2.5."
+  else
+    setup do
+      @lock = Resque::Scheduler::Lock::Resilient.new('test_resilient_lock')
+    end
+
+    teardown do
+      @lock.release!
+    end
+
+    test 'you should not have the lock if someone else holds it' do
+      lock_is_not_held(@lock)
+
+      assert !@lock.locked?, 'you should not have the lock'
+    end
+
+    test 'you should not be able to acquire the lock if someone else holds it' do
+      lock_is_not_held(@lock)
+
+      assert !@lock.acquire!
+    end
+
+    test "the lock should receive a TTL on acquiring" do
+      @lock.acquire!
+
+      assert Resque.redis.ttl(@lock.key) > 0, "lock should expire"
+    end
+
+    test 'releasing should release the master lock' do
+      assert @lock.acquire!, 'should have acquired the master lock'
+      assert @lock.locked?, 'should be locked'
+
+      @lock.release!
+
+      assert !@lock.locked?, 'should not be locked'
+    end
+
+    test 'checking the lock should increase the TTL if we hold it' do
+      @lock.acquire!
+      Resque.redis.setex(@lock.key, 10, @lock.value)
+
+      @lock.locked?
+
+      assert Resque.redis.ttl(@lock.key) > 10, "TTL should have been updated"
+    end
+
+    test 'checking the lock should not increase the TTL if we do not hold it' do
+      Resque.redis.setex(@lock.key, 10, @lock.value)
+      lock_is_not_held(@lock)
+
+      @lock.locked?
+
+      assert Resque.redis.ttl(@lock.key) <= 10, "TTL should not have been updated"
+    end
+  end
+end

data/test/scheduler_test.rb

@@ -4,8 +4,7 @@ context "Resque::Scheduler" do
 
   setup do
     Resque::Scheduler.dynamic = false
-    Resque.redis.del(:schedules)
-    Resque.redis.del(:schedules_changed)
+    Resque.redis.flushall
     Resque::Scheduler.mute = true
     Resque::Scheduler.clear_schedule!
     Resque::Scheduler.send(:class_variable_set, :@@scheduled_jobs, {})
@@ -72,15 +71,15 @@ context "Resque::Scheduler" do
     assert Resque::Scheduler.scheduled_jobs.include?("some_ivar_job2")
   end
 
-  test "load_schedule_job loads a schedule" do 
+  test "load_schedule_job loads a schedule" do
     Resque::Scheduler.load_schedule_job("some_ivar_job", {'cron' => "* * * * *", 'class' => 'SomeIvarJob', 'args' => "/tmp"})
 
     assert_equal(1, Resque::Scheduler.rufus_scheduler.all_jobs.size)
     assert_equal(1, Resque::Scheduler.scheduled_jobs.size)
     assert Resque::Scheduler.scheduled_jobs.keys.include?("some_ivar_job")
   end
-  
-  test "load_schedule_job with every with options" do 
+
+  test "load_schedule_job with every with options" do
     Resque::Scheduler.load_schedule_job("some_ivar_job", {'every' => ['30s', {'first_in' => '60s'}], 'class' => 'SomeIvarJob', 'args' => "/tmp"})
 
     assert_equal(1, Resque::Scheduler.rufus_scheduler.all_jobs.size)
@@ -88,8 +87,8 @@ context "Resque::Scheduler" do
     assert Resque::Scheduler.scheduled_jobs.keys.include?("some_ivar_job")
     assert Resque::Scheduler.scheduled_jobs["some_ivar_job"].params.keys.include?(:first_in)
   end
-  
-  test "load_schedule_job with cron with options" do 
+
+  test "load_schedule_job with cron with options" do
     Resque::Scheduler.load_schedule_job("some_ivar_job", {'cron' => ['* * * * *', {'allow_overlapping' => 'true'}], 'class' => 'SomeIvarJob', 'args' => "/tmp"})
 
     assert_equal(1, Resque::Scheduler.rufus_scheduler.all_jobs.size)
@@ -243,5 +242,4 @@ context "Resque::Scheduler" do
       Resque::Plugin.lint(ResqueScheduler)
     end
   end
-
 end

data/test/support/redis_instance.rb

@@ -0,0 +1,129 @@
+require 'socket'
+require 'timeout'
+require 'fileutils'
+
+class RedisInstance
+  class << self
+    @running = false
+    @port = nil
+    @pid = nil
+
+    def run_if_needed!
+      run! unless running?
+    end
+
+    def run!
+      ensure_pid_directory
+      reassign_redis_clients
+      start_redis_server
+
+      if $?.success?
+        wait_for_pid
+        puts "Booted isolated Redis on port #{port} with PID #{pid}."
+
+        wait_for_redis_boot
+
+        # Ensure we tear down Redis on Ctrl+C / test failure.
+        at_exit do
+          RedisInstance.stop!
+        end
+      else
+        raise "Failed to start Redis on port #{port}."
+      end
+
+      @running = true
+    end
+
+    def stop!
+      $stdout.puts "Sending TERM to Redis (#{pid})..."
+      Process.kill('TERM', pid)
+
+      @port = nil
+      @running = false
+      @pid = nil
+    end
+
+    def running?
+      @running
+    end
+
+  private
+
+    def wait_for_redis_boot
+      Timeout::timeout(10) do
+        begin
+          while Resque.redis.ping != 'PONG'
+          end
+        rescue
+          # silence all errors
+        end
+      end
+    end
+
+    def ensure_pid_directory
+      FileUtils.mkdir_p(File.dirname(pid_file))
+    end
+
+    def reassign_redis_clients
+      Resque.redis = Redis.new(:hostname => '127.0.0.1', :port => port, :thread_safe => true)
+    end
+
+    def start_redis_server
+      IO.popen("redis-server -", "w+") do |server|
+        server.write(config)
+        server.close_write
+      end
+    end
+
+    def pid
+      @pid ||= File.read(pid_file).to_i
+    end
+
+    def wait_for_pid
+      Timeout::timeout(10) do
+        while !File.exist?(pid_file)
+        end
+      end
+    end
+
+    def port
+      @port ||= random_port
+    end
+
+    def pid_file
+      "/tmp/redis-scheduler-test.pid"
+    end
+
+    def config
+      <<-EOF
+        daemonize yes
+        pidfile #{pid_file}
+        port #{port}
+      EOF
+    end
+
+    # Returns a random port in the upper (10000-65535) range.
+    def random_port
+      ports = (10000..65535).to_a
+
+      loop do
+        port = ports[rand(ports.size)]
+        return port if port_available?('127.0.0.1', port)
+      end
+    end
+
+    def port_available?(ip, port, seconds=1)
+      Timeout::timeout(seconds) do
+        begin
+          TCPSocket.new(ip, port).close
+          false
+        rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
+          true
+        end
+      end
+    rescue Timeout::Error
+      true
+    end
+  end
+end
+

data/test/test_helper.rb

@@ -23,10 +23,10 @@ if !system("which redis-server")
 end
 
 
-#
-# start our own redis when the tests start,
-# kill it when they end
-#
+# Start our own Redis when the tests start. RedisInstance will take care of
+# starting and stopping.
+require File.dirname(__FILE__) + '/support/redis_instance'
+RedisInstance.run!
 
 at_exit do
   next if $!
@@ -37,17 +37,9 @@ at_exit do
     exit_code = Test::Unit::AutoRunner.run
   end
 
-  pid = `ps -e -o pid,command | grep [r]edis-test`.split(" ")[0]
-  puts "Killing test redis server..."
-  `rm -f #{dir}/dump.rdb`
-  Process.kill("KILL", pid.to_i)
   exit exit_code
 end
 
-puts "Starting redis for testing at localhost:9736..."
-`redis-server #{dir}/redis-test.conf`
-Resque.redis = 'localhost:9736'
-
 ##
 # test/spec/mini 3
 # http://gist.github.com/25455
@@ -72,6 +64,11 @@ class FakeCustomJobClass
   def self.scheduled(queue, klass, *args); end
 end
 
+class FakeCustomJobClassEnqueueAt
+  @queue = :test
+  def self.scheduled(queue, klass, *args); end
+end
+
 class SomeJob
   def self.perform(repo_id, path)
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: resque-scheduler
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-04 00:00:00.000000000 Z
+date: 2013-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -91,6 +91,11 @@ files:
 - README.markdown
 - Rakefile
 - lib/resque/scheduler.rb
+- lib/resque/scheduler/lock.rb
+- lib/resque/scheduler/lock/base.rb
+- lib/resque/scheduler/lock/basic.rb
+- lib/resque/scheduler/lock/resilient.rb
+- lib/resque/scheduler_locking.rb
 - lib/resque_scheduler.rb
 - lib/resque_scheduler/plugin.rb
 - lib/resque_scheduler/server.rb
@@ -106,7 +111,9 @@ files:
 - test/resque-web_test.rb
 - test/scheduler_args_test.rb
 - test/scheduler_hooks_test.rb
+- test/scheduler_locking_test.rb
 - test/scheduler_test.rb
+- test/support/redis_instance.rb
 - test/test_helper.rb
 homepage: http://github.com/bvandenbos/resque-scheduler
 licenses: []
@@ -128,7 +135,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: 1.3.6
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.21
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Light weight job scheduling on top of Resque