resque-batched-job 0.6.0 → 1.0.0

data/Rakefile

@@ -2,6 +2,8 @@ require 'rubygems'
 require 'rake'
 require 'rake/testtask'
 
+require File.dirname(__FILE__) + '/lib/resque/plugins/batched_job/version'
+
 task :default => :test
 
 Rake::TestTask.new do |task|
@@ -9,16 +11,29 @@ Rake::TestTask.new do |task|
   task.verbose = true
 end
 
-desc "Publish gem and source."
-task :publish => :build do
-  require File.dirname(__FILE__) + '/lib/resque/plugins/batched_job/version'
-  sh "gem push resque-batched-job-#{Resque::Plugins::BatchedJob::VERSION}.gem"
-  sh "git tag v#{Resque::Plugins::BatchedJob::VERSION}"
+desc "Publish RubyGem and source."
+task :publish => [:build, :tag] do
   sh "git push origin v#{Resque::Plugins::BatchedJob::VERSION}"
   sh "git push origin master"
+  sh "gem push resque-batched-job-#{Resque::Plugins::BatchedJob::VERSION}.gem"
+end
+
+desc "Tag project with current version."
+task :tag do
+  sh "git tag v#{Resque::Plugins::BatchedJob::VERSION}"
 end
 
-desc "Build gem."
+desc "Build RubyGem."
 task :build do
   sh "gem build resque-batched-job.gemspec"
 end
+
+desc "View changelog"
+task :changelog do
+  tags = `git tag`.split("\n").reverse
+
+  tags.each_slice(2) do |tags|
+    puts "========== #{tags[1]}..#{tags[0]} =========="
+    sh "git log --pretty=format:'%h : %s' --graph #{tags[1]}..#{tags[0]}"
+  end
+end

data/lib/resque/plugins/batched_job/version.rb

@@ -1,7 +1,7 @@
 module Resque
   module Plugins
     module BatchedJob
-      VERSION = '0.6.0'
+      VERSION = '1.0.0'
     end
   end
 end

data/lib/resque/plugins/batched_job.rb

@@ -22,14 +22,17 @@ module Resque
 
       # Batch the job.  The first argument of a batched job, is the batch id.
       def after_enqueue_batch(id, *args)
-        redis.rpush(batch(id), encode(:class => self, :args => args))
+        mutex(id) do |bid|
+          redis.rpush(bid, encode(:class => self, :args => args))
+        end
       end
 
       # After the job is performed, remove it from the batched job list.  If the
       # current job is the last in the batch to be performed, invoke the after_batch
       # hooks.
       def after_perform_batch(id, *args)
-        redis.lrem(batch(id), 1, encode(:class => self, :args => args))
+        remove_batched_job(id, *args)
+
         if batch_complete?(id)
           after_batch_hooks = Resque::Plugin.after_batch_hooks(self)
           after_batch_hooks.each do |hook|
@@ -41,13 +44,67 @@ module Resque
       # Checks the size of the batched job list and returns true if the list is
       # empty or if the key does not exist.
       def batch_complete?(id)
-        redis.llen(batch(id)) == 0
+        mutex(id) do |bid| 
+          redis.llen(bid) == 0
+        end
       end
 
       def batch_exist?(id)
-        redis.exists(batch(id))
+        mutex(id) do |bid| 
+          redis.exists(bid)
+        end
       end
 
+      # Remove a job from the batch list.
+      def remove_batched_job(id, *args)
+        mutex(id) do |bid|
+          redis.lrem(bid, 1, encode(:class => self, :args => args))
+        end
+      end
+
+      private
+
+        # Lock a batch key before executing Redis commands.  This will ensure
+        # no race conditions occur when modifying batch information.  Here is
+        # an example of how this works.  See http://redis.io/commands/setnx for
+        # more information.
+        # 
+        # * Job2 sends SETNX batch:123:lock in order to aquire a lock.
+        # * Job1 still has the key locked, so Job2 continues into the loop.
+        # * Job2 sends GET to aquire the lock timestamp.
+        # * If the timestamp does not exist (Job1 released the lock), Job2 
+        #   attemps to start from the beginning again.
+        # * If the timestamp exists and has not expired, Job2 sleeps for a 
+        #   moment and then retries from the start.
+        # * If the timestamp exists and has expired, Job2 sends GETSET to aquire
+        #   a lock.  This returns the previous value of the lock.
+        # * If the previous timestamp has not expired, another process was faster
+        #   and aquired the lock.  This means Job2 has to start from the beginnig.
+        # * If the previous timestamp is still expired the lock has been set and
+        #   processing can continue safely
+        def mutex(id, &block)
+          is_expired = lambda do |locked_at|
+            locked_at.to_f < Time.now.to_f
+          end
+          bid   = batch(id)
+          _key_ = "#{bid}:lock"
+
+          until redis.setnx(_key_, Time.now.to_f + 0.5)
+            next unless timestamp = redis.get(_key_)
+
+            unless is_expired.call(timestamp)
+              sleep(0.1)
+              next
+            end
+
+            break unless timestamp = redis.getset(_key_, Time.now.to_f + 0.5)
+            break if is_expired.call(timestamp)
+          end
+          yield(bid)
+        ensure
+          redis.del(_key_)
+        end
+
     end
 
   end

data/test/test_batched_job.rb

@@ -1,57 +1,123 @@
-require 'rubygems'
-require 'test/unit'
-require 'resque'
-require 'resque-batched-job'
-
-class Job
-  extend Resque::Plugins::BatchedJob
-  @queue = :test
-  
-  def self.perform(batch_id, arg)
-  end
-  
-  def self.after_batch_hook(batch_id, arg)
-    $batch_complete = true
-  end
-  
-end
+require File.dirname(__FILE__) + '/test_helper'
 
 class BatchedJobTest < Test::Unit::TestCase
-  
+
   def setup
     $batch_complete = false
     @batch_id = :foo
     @batch = "batch:#{@batch_id}"
-    @cnt = 5
-    @cnt.times { Resque.enqueue(Job, @batch_id, "arg#{rand(100)}") }
   end
-  
+
   def teardown
     redis.del(@batch)
     redis.del("queue:test")
+    redis.del("#{@batch}:lock")
   end
-  
+
   def test_list
     assert_nothing_raised do
       Resque::Plugin.lint(Resque::Plugins::BatchedJob)
     end
   end
-  
+
   def test_batch_key
+    assert_nothing_raised do
+      Resque.enqueue(Job, @batch_id, 'foobar')
+    end
     assert_equal(@batch, Job.batch(@batch_id))
   end
-  
+
+  # Ensure the length of the Redis list matches the number of jobs we enqueue.
   def test_batch_size
-    # assert_equal(@cnt, redis.smembers(@batch).size)
-    assert_equal(@cnt, redis.llen(@batch))
+    assert_nothing_raised do
+      5.times { Resque.enqueue(Job, @batch_id, "arg#{rand(100)}") }
+    end
+    assert_equal(5, redis.llen(@batch))
   end
-  
+
+  # Make sure the after_batch hook is fired
   def test_batch_hook
+    assert_equal(false, Job.batch_exist?(@batch_id))
+    
+    assert_nothing_raised do
+      5.times { Resque.enqueue(Job, @batch_id, "arg#{rand(100)}") }
+    end
+
+    assert_equal(false, $batch_complete)
+    assert_equal(false, Job.batch_complete?(@batch_id))
+    assert(Job.batch_exist?(@batch_id))
+
+    assert_nothing_raised do
+      4.times { Resque.reserve(:test).perform }
+    end
+
+    assert_equal(false, $batch_complete)
+    assert_equal(false, Job.batch_complete?(@batch_id))
+    assert(Job.batch_exist?(@batch_id))
+
+    assert_nothing_raised do
+      Resque.reserve(:test).perform
+    end
+
+    assert($batch_complete)
+    assert(Job.batch_complete?(@batch_id))
+    assert_equal(false, Job.batch_exist?(@batch_id))
+  end
+
+  # Test that jobs with identical args behave properly.
+  def test_duplicate_args
+    assert_equal(false, Job.batch_exist?(@batch_id))
+    
+    assert_nothing_raised do
+      5.times { Resque.enqueue(JobWithoutArgs, @batch_id) }
+    end
+
+    assert_equal(false, $batch_complete)
+    assert_equal(false, Job.batch_complete?(@batch_id))
+    assert(Job.batch_exist?(@batch_id))
+
+    assert_nothing_raised do
+      2.times { Resque.reserve(:test).perform }
+    end
+
+    assert_equal(false, $batch_complete)
+    assert_equal(false, Job.batch_complete?(@batch_id))
+    assert(Job.batch_exist?(@batch_id))
+
     assert_nothing_raised do
-      @cnt.times { Resque.reserve(:test).perform }
+      3.times { Resque.reserve(:test).perform }
     end
+
     assert($batch_complete)
-    assert_equal(false, redis.exists(@batch))
+    assert(Job.batch_complete?(@batch_id))
+    assert_equal(false, Job.batch_exist?(@batch_id))
+  end
+
+  # Make sure the block is executed and the lock is removed.
+  def test_mutex
+    Job.send(:mutex, @batch_id) do
+      assert(true)
+    end
+    assert_equal(false, redis.exists("#{@batch}:lock"))
+  end
+
+  # Make sure no race conditions occur.
+  def test_locking
+    threads = []
+    x, y = 10, 5
+
+    x.times do
+      threads << Thread.new do
+        y.times do
+          Job.send(:mutex, @batch_id) do
+            redis.incr(@batch)
+          end
+        end
+      end
+    end
+    threads.each { |t| t.join }
+
+    assert_equal(x * y, Integer(redis.get(@batch)))
   end
 
   private
@@ -59,5 +125,5 @@ class BatchedJobTest < Test::Unit::TestCase
     def redis
       Resque.redis
     end
-    
+
 end

data/test/test_helper.rb

@@ -0,0 +1,32 @@
+require 'rubygems'
+require 'test/unit'
+require 'thread'
+
+require 'resque'
+require File.dirname(__FILE__) + '/../lib/resque-batched-job'
+
+class Job
+  extend Resque::Plugins::BatchedJob
+  @queue = :test
+
+  def self.perform(batch_id, arg)
+  end
+
+  def self.after_batch_hook(batch_id, arg)
+    $batch_complete = true
+  end
+
+end
+
+class JobWithoutArgs
+  extend Resque::Plugins::BatchedJob
+  @queue = :test
+
+  def self.perform(id)
+  end
+
+  def self.after_batch_hook(id)
+    $batch_complete = true
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: resque-batched-job
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 1.0.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-09 00:00:00.000000000Z
+date: 2011-12-11 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70335551372100 !ruby/object:Gem::Requirement
+  requirement: &70295155687280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,7 +21,7 @@ dependencies:
         version: 1.10.0
   type: :runtime
   prerelease: false
-  version_requirements: *70335551372100
+  version_requirements: *70295155687280
 description: ! "  Resque plugin for batching jobs. When a batch/group of jobs are
   complete, \nadditional work can be performed usings batch hooks.\n"
 email: dan@dj-agiledev.com
@@ -36,6 +36,7 @@ files:
 - lib/resque/plugins/batched_job.rb
 - lib/resque-batched-job.rb
 - test/test_batched_job.rb
+- test/test_helper.rb
 homepage: https://github.com/drfeelngood/resque-batched-job
 licenses: []
 post_install_message: 
@@ -62,3 +63,4 @@ specification_version: 3
 summary: Resque plugin
 test_files:
 - test/test_batched_job.rb
+- test/test_helper.rb