resque-concurrent-restriction 0.5.8 → 0.5.9

data/CHANGELOG

@@ -1,3 +1,9 @@
+0.5.9 (06/04/2013)
+------------------
+
+Lower slice-size used during "reset_restrictions" from 100K to 10K (it was causing segfaults when testing) <b9a7b57>
+Merge pull request #5 from jzaleski/stash_restrictions_more_aggressively <a0c051f>
+
 0.5.8 (06/07/2012)
 ------------------
 

data/lib/resque/plugins/concurrent_restriction/concurrent_restriction_job.rb

@@ -7,6 +7,10 @@
 #      # How many times to try to get a lock before giving up
 #      # Worker stays busy for: 2^tries * rand(100) * 0.001 (~30s-3000s)
 #      config.lock_tries = 10
+#      # Try this many times to reserve a job from a queue. Also, the maximum
+#      # number of jobs to move to the restricted queue during this process
+#      # before giving up
+#      config.reserve_queued_job_attempts = 10
 #      # Try to pick jobs off of the restricted queue before normal queues
 #      config.restricted_before_queued = true
 #    end
@@ -18,12 +22,13 @@ module Resque
       # Allows configuring via class accessors
       class << self
         # optional
-        attr_accessor :lock_timeout, :lock_tries, :restricted_before_queued
+        attr_accessor :lock_timeout, :lock_tries, :reserve_queued_job_attempts, :restricted_before_queued
       end
 
       # default values
       self.lock_timeout = 60
       self.lock_tries = 15
+      self.reserve_queued_job_attempts = 1
       self.restricted_before_queued = false
 
       # Allows configuring via class accessors
@@ -443,14 +448,14 @@ module Resque
         counts_reset = 0
         count_keys = Resque.redis.keys("concurrent.count.*")
         if count_keys.size > 0
-          count_keys.each_slice(100000) do |key_slice|
+          count_keys.each_slice(10000) do |key_slice|
             counts_reset += Resque.redis.del(*key_slice)
           end
         end
 
         runnable_keys = Resque.redis.keys("concurrent.runnable*")
         if runnable_keys.size > 0
-          runnable_keys.each_slice(100000) do |runnable_slice|
+          runnable_keys.each_slice(10000) do |runnable_slice|
             Resque.redis.del(*runnable_slice)
           end
         end

data/lib/resque/plugins/concurrent_restriction/resque_worker_extension.rb

@@ -67,23 +67,27 @@ module Resque
         end
 
         def get_queued_job(queue)
-          resque_job = reserve_without_restriction(queue)
+          # Bounded retry
+          1.upto(ConcurrentRestriction.reserve_queued_job_attempts) do |i|
+            resque_job = reserve_without_restriction(queue)
 
-          if resque_job
-            # If there is a job on regular queues, then only run it if its not restricted
+            # Short-curcuit if a job was not found
+            return if resque_job.nil?
 
+            # If there is a job on regular queues, then only run it if its not restricted
             job_class = resque_job.payload_class
             job_args = resque_job.args
 
-            # return to work on job if not a restricted job
-            if job_class.is_a?(ConcurrentRestriction)
-              # Move on to next if job is restricted
-              # If job is runnable, we keep the lock until done_working
-              resque_job = nil if job_class.stash_if_restricted(resque_job)
-            end
+            # Return to work on job if not a restricted job
+            return resque_job unless job_class.is_a?(ConcurrentRestriction)
+
+            # Keep trying if job is restricted. If job is runnable, we keep the lock until
+            # done_working
+            return resque_job unless job_class.stash_if_restricted(resque_job)
           end
 
-          return resque_job
+          # Safety net, here in case we hit the upper bound and there are still queued items
+          return nil
         end
 
       end

data/lib/resque/plugins/concurrent_restriction/version.rb

@@ -1,7 +1,7 @@
 module Resque
   module Plugins
     module ConcurrentRestriction
-      VERSION = "0.5.8"
+      VERSION = "0.5.9"
     end
   end
 end

data/spec/concurrent_restriction_job_spec.rb

@@ -28,6 +28,16 @@ describe Resque::Plugins::ConcurrentRestriction do
       Resque::Plugins::ConcurrentRestriction.lock_timeout.should == 60
     end
 
+    it "should allow setting/getting global config for reserve_queued_job_attempts" do
+      Resque::Plugins::ConcurrentRestriction.reserve_queued_job_attempts.should == 1
+      Resque::Plugins::ConcurrentRestriction.configure do |config|
+        config.reserve_queued_job_attempts = 5
+      end
+      Resque::Plugins::ConcurrentRestriction.reserve_queued_job_attempts.should == 5
+      Resque::Plugins::ConcurrentRestriction.reserve_queued_job_attempts = 3
+      Resque::Plugins::ConcurrentRestriction.reserve_queued_job_attempts.should == 3
+    end
+
     it "should allow setting/getting global config for restricted_before_queued" do
       Resque::Plugins::ConcurrentRestriction.restricted_before_queued.should == false
       Resque::Plugins::ConcurrentRestriction.configure do |config|
@@ -591,13 +601,13 @@ describe Resque::Plugins::ConcurrentRestriction do
       # It might be better to actually populate redis with a bunch keys but that makes the test pretty slow
 
       # we have to keep this splat limitation in mind when populating test data, too
-      concurrent_count_keys = 200001.times.collect{ |i| ["concurrent.count.#{i}", "#{i}"] }.flatten
-      concurrent_count_keys.each_slice(100000) do |slice|
+      concurrent_count_keys = 20001.times.collect{ |i| ["concurrent.count.#{i}", "#{i}"] }.flatten
+      concurrent_count_keys.each_slice(10000) do |slice|
         Resque.redis.mset *slice
       end
 
-      concurrent_runnable_keys = 200001.times.collect{ |i| ["concurrent.runnable.#{i}", "#{i}"] }.flatten
-      concurrent_runnable_keys.each_slice(100000) do |slice|
+      concurrent_runnable_keys = 20001.times.collect{ |i| ["concurrent.runnable.#{i}", "#{i}"] }.flatten
+      concurrent_runnable_keys.each_slice(10000) do |slice|
         Resque.redis.mset *slice
       end
 
@@ -605,7 +615,7 @@ describe Resque::Plugins::ConcurrentRestriction do
 
       lambda{ return_value = ConcurrentRestrictionJob.reset_restrictions }.should_not raise_exception
 
-      return_value.should == [200001, 0]
+      return_value.should == [20001, 0]
     end
 
   end

data/spec/resque_worker_extensions_spec.rb

@@ -259,4 +259,19 @@ describe Resque::Plugins::ConcurrentRestriction::Worker do
     RestrictionJob.total_run_count.should == 3
   end
 
+  it "should move multiple items to the restricted queue each iteration" do
+    RestrictionJob.set_running_count(RestrictionJob.tracking_key, 99)
+
+    5.times {|i| Resque.enqueue(RestrictionJob, :queue => :normal)}
+    Resque.size(:normal).should == 5
+    RestrictionJob.restriction_queue(RestrictionJob.tracking_key, :normal).size.should == 0
+
+    Resque::Plugins::ConcurrentRestriction.reserve_queued_job_attempts = 3
+
+    run_resque_queue(:normal)
+    RestrictionJob.run_count.should == 0
+    Resque.size(:normal).should == 2
+    RestrictionJob.restriction_queue(RestrictionJob.tracking_key, :normal).size.should == 3
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: resque-concurrent-restriction
 version: !ruby/object:Gem::Version
-  version: 0.5.8
+  version: 0.5.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-07 00:00:00.000000000 Z
+date: 2013-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
@@ -135,7 +135,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: resque-concurrent-restriction
-rubygems_version: 1.8.21
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: A resque plugin for limiting how many of a specific job can run concurrently