resque-concurrent-restriction 0.5.5 → 0.5.6

data/lib/resque/plugins/concurrent_restriction/concurrent_restriction_job.rb

@@ -4,6 +4,9 @@
 #    Resque::Plugins::ConcurrentRestriction.configure do |config|
 #      # The lock timeout for the restriction queue lock
 #      config.lock_timeout = 60
+#      # How many times to try to get a lock before giving up
+#      # Worker stays busy for: 2^tries * rand(100) * 0.001 (~30s-3000s)
+#      config.lock_tries = 10
 #      # Try to pick jobs off of the restricted queue before normal queues
 #      config.restricted_before_queued = true
 #    end
@@ -15,11 +18,12 @@ module Resque
       # Allows configuring via class accessors
       class << self
         # optional
-        attr_accessor :lock_timeout, :restricted_before_queued
+        attr_accessor :lock_timeout, :lock_tries, :restricted_before_queued
       end
 
       # default values
       self.lock_timeout = 60
+      self.lock_tries = 15
       self.restricted_before_queued = false
 
       # Allows configuring via class accessors
@@ -341,25 +345,30 @@ module Resque
 
       # Uses a lock to ensure that a sequence of redis operations happen atomically
       # We don't use watch/multi/exec as it doesn't work in a DistributedRedis setup
-      def run_atomically(lock_key)
-        trying = true
+      def run_atomically(lock_key, tries=ConcurrentRestriction.lock_tries)
+        acquired_lock = false
         exp_backoff = 1
 
-        while trying do
+        tries.times do
           lock_expiration = Time.now.to_i + ConcurrentRestriction.lock_timeout
+          p [Time.now.to_f, Process.pid, :start]
           if acquire_lock(lock_key, lock_expiration)
+            p [Time.now.to_f, Process.pid, :acquired]
+            acquired_lock = true
             begin
               yield
+              p [Time.now.to_f, Process.pid, :end]              
             ensure
               release_lock(lock_key, lock_expiration)
             end
-            trying = false
+            break
           else
-            sleep(rand(1000) * 0.0001 * exp_backoff)
+            sleep(rand(100) * 0.001 * exp_backoff)
             exp_backoff *= 2
           end
         end
-
+        
+        return acquired_lock
       end
 
       # Pushes the job to restriction queue if it is restricted
@@ -368,11 +377,11 @@ module Resque
       # see a lower value and run their job.  This count gets decremented by call
       # to release_restriction when job completes
       def stash_if_restricted(job)
-        restricted = false
+        restricted = nil
         tracking_key = tracking_key(*job.args)
         lock_key = lock_key(tracking_key)
 
-        run_atomically(lock_key) do
+        did_run = run_atomically(lock_key) do
 
           restricted = restricted?(tracking_key)
           if restricted
@@ -382,7 +391,15 @@ module Resque
           end
 
         end
-
+        
+        # if run_atomically fails to acquire the lock, we need to put
+        # the job back on the queue for processing later and act restricted
+        # upstack so nothing gets run
+        if !did_run
+          restricted = true
+          job.recreate
+        end
+        
         return restricted
       end
 

data/lib/resque/plugins/concurrent_restriction/version.rb

@@ -1,7 +1,7 @@
 module Resque
   module Plugins
     module ConcurrentRestriction
-      VERSION = "0.5.5"
+      VERSION = "0.5.6"
     end
   end
 end

data/spec/concurrent_restriction_job_spec.rb

@@ -132,6 +132,37 @@ describe Resque::Plugins::ConcurrentRestriction do
       t1.join
       counter.should == "second"
     end
+    
+    it "should fail if can't acquire lock within tries" do
+      did_run1 = did_run2 = false
+
+      t1 = Thread.new do
+        ConcurrentRestrictionJob.run_atomically("some_lock_key") do
+          sleep 0.2
+        end
+      end
+
+      sleep 0.1
+      t1.alive?.should == true
+      
+      t2 = Thread.new do
+        ConcurrentRestrictionJob.run_atomically("some_lock_key", 2) do
+          did_run1 = true
+        end
+      end
+      t2.join
+      
+      t3 = Thread.new do
+        ConcurrentRestrictionJob.run_atomically("some_lock_key", 100) do
+          did_run2 = true
+        end
+      end
+      t3.join
+
+      t1.join
+      did_run1.should be_false
+      did_run2.should be_true
+    end
 
   end
 
@@ -402,6 +433,20 @@ describe Resque::Plugins::ConcurrentRestriction do
       ConcurrentRestrictionJob.runnables("somequeue2").sort.should == [ConcurrentRestrictionJob.tracking_key]
 
     end
+    
+    it "should repush job and return true if it can't acquire a lock" do
+      old = Resque::Plugins::ConcurrentRestriction.lock_tries
+      begin
+        Resque::Plugins::ConcurrentRestriction.lock_tries = 0
+  
+        job = Resque::Job.new("somequeue", {"class" => "ConcurrentRestrictionJob", "args" => []})
+        ConcurrentRestrictionJob.stash_if_restricted(job).should == true
+        Resque.peek("somequeue").should == {"class" => "ConcurrentRestrictionJob", "args" => []}
+      ensure
+        Resque::Plugins::ConcurrentRestriction.lock_tries = old
+      end
+      
+    end
 
   end
 
@@ -411,6 +456,23 @@ describe Resque::Plugins::ConcurrentRestriction do
       ConcurrentRestrictionJob.next_runnable_job('somequeue').should be_nil
     end
 
+    it "should return nil and not pop from queue if cannot acquire lock" do
+      job1 = Resque::Job.new("somequeue", {"class" => "ConcurrentRestrictionJob", "args" => []})
+      ConcurrentRestrictionJob.set_running_count(ConcurrentRestrictionJob.tracking_key, 99)
+      ConcurrentRestrictionJob.stash_if_restricted(job1)
+      ConcurrentRestrictionJob.set_running_count(ConcurrentRestrictionJob.tracking_key, 0)
+
+      old = Resque::Plugins::ConcurrentRestriction.lock_tries
+      begin
+        Resque::Plugins::ConcurrentRestriction.lock_tries = 0
+  
+        ConcurrentRestrictionJob.next_runnable_job('somequeue').should be_nil
+        ConcurrentRestrictionJob.restriction_queue(ConcurrentRestrictionJob.tracking_key, "somequeue").should == [job1]
+      ensure
+        Resque::Plugins::ConcurrentRestriction.lock_tries = old
+      end
+    end
+    
     it "should not get a job if nothing runnable" do
       job1 = Resque::Job.new("somequeue", {"class" => "ConcurrentRestrictionJob", "args" => []})
       ConcurrentRestrictionJob.set_running_count(ConcurrentRestrictionJob.tracking_key, 99)
@@ -467,6 +529,21 @@ describe Resque::Plugins::ConcurrentRestriction do
       ConcurrentRestrictionJob.running_count(ConcurrentRestrictionJob.tracking_key).should == 0
     end
 
+    it "should do nothing if cannot acquire lock" do
+      ConcurrentRestrictionJob.set_running_count(ConcurrentRestrictionJob.tracking_key, 1)
+      job = Resque::Job.new("somequeue", {"class" => "ConcurrentRestrictionJob", "args" => []})
+
+      old = Resque::Plugins::ConcurrentRestriction.lock_tries
+      begin
+        Resque::Plugins::ConcurrentRestriction.lock_tries = 0
+  
+        ConcurrentRestrictionJob.release_restriction(job)
+        ConcurrentRestrictionJob.running_count(ConcurrentRestrictionJob.tracking_key).should == 1
+      ensure
+        Resque::Plugins::ConcurrentRestriction.lock_tries = old
+      end
+    end
+    
   end
 
   context "#reset_restrictions" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: resque-concurrent-restriction
 version: !ruby/object:Gem::Version
-  version: 0.5.5
+  version: 0.5.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-23 00:00:00.000000000 Z
+date: 2012-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque