resque-concurrent-restriction 0.5.1 → 0.5.2

data/lib/resque/plugins/concurrent_restriction/concurrent_restriction_job.rb

@@ -74,6 +74,12 @@ module Resque
         "concurrent.queue.#{queue}.#{parts[2..-1].join('.')}"
       end
 
+      # The redis key used to store the aggregate number of jobs
+      # in restriction queues by queue name
+      def queue_count_key
+        "concurrent.queue_counts"
+      end
+
       def restriction_queue_availability_key(tracking_key)
         parts = tracking_key.split(".")
         "concurrent.queue_availability.#{parts[2..-1].join('.')}"
@@ -139,6 +145,7 @@ module Resque
           else raise "Invalid location to ConcurrentRestriction.push_to_restriction_queue"
         end
 
+        increment_queue_count(job.queue)
         update_queues_available(tracking_key, job.queue, :add)
         mark_runnable(tracking_key, false)
       end
@@ -154,6 +161,8 @@ module Resque
           clear_runnable(tracking_key, queue)
         end
 
+        decrement_queue_count(queue)
+
         # increment by one to indicate that we are running
         # do this before update_queues_available so that the current queue gets cleaned
         increment_running_count(tracking_key) if str
@@ -209,6 +218,26 @@ module Resque
         return restricted
       end
 
+      def increment_queue_count(queue, by=1)
+        value = Resque.redis.hincrby(queue_count_key, queue, by)
+        return value
+      end
+
+      def decrement_queue_count(queue, by=1)
+        value = Resque.redis.hincrby(queue_count_key, queue, -by)
+        return value
+      end
+
+      def queue_counts
+        value = Resque.redis.hgetall(queue_count_key)
+        value = Hash[*value.collect {|k, v| [k, v.to_i] }.flatten]
+        return value
+      end
+
+      def set_queue_count(queue, count)
+        Resque.redis.hset(queue_count_key, queue, count)
+      end
+
       def runnable?(tracking_key, queue)
         Resque.redis.sismember(runnables_key(queue), tracking_key)
       end
@@ -375,14 +404,18 @@ module Resque
         runnable_keys = Resque.redis.keys("concurrent.runnable*")
         Resque.redis.del(*runnable_keys) if runnable_keys.size > 0
 
+        Resque.redis.del(queue_count_key)
         queues_enabled = 0
         queue_keys = Resque.redis.keys("concurrent.queue.*")
         queue_keys.each do |k|
-          if Resque.redis.llen(k) > 0
+          len = Resque.redis.llen(k)
+          if len > 0
             parts = k.split(".")
             queue = parts[2]
             ident = parts[3..-1].join('.')
             tracking_key = "concurrent.tracking.#{ident}"
+
+            increment_queue_count(queue, len)
             update_queues_available(tracking_key, queue, :add)
             mark_runnable(tracking_key, true)
             queues_enabled += 1
@@ -393,46 +426,46 @@ module Resque
         
       end
 
-      def stats
-        results = {}
+      def stats(extended=false)
+        result = {}
 
-        queue_keys = Resque.redis.keys("concurrent.queue.*")
+        result[:queues] = queue_counts
 
-        queue_sizes = {}
-        ident_sizes = {}
-        queue_keys.each do |k|
-          parts = k.split(".")
-          ident = parts[3..-1].join(".")
-          queue_name = parts[2]
-          size = Resque.redis.llen(k)
-          queue_sizes[queue_name] ||= 0
-          queue_sizes[queue_name] += size
-          ident_sizes[ident] ||= 0
-          ident_sizes[ident] += size
-        end
+        if extended
+          ident_sizes = {}
+          queue_keys = Resque.redis.keys("concurrent.queue.*")
+          queue_keys.each do |k|
+            parts = k.split(".")
+            ident = parts[3..-1].join(".")
+            queue_name = parts[2]
+            size = Resque.redis.llen(k)
+            ident_sizes[ident] ||= {}
+            ident_sizes[ident][queue_name] ||= 0
+            ident_sizes[ident][queue_name] += size
+          end
 
-        count_keys = Resque.redis.keys("concurrent.count.*")
-        running_counts = {}
-        count_keys.each do |k|
-          parts = k.split(".")
-          ident = parts[2..-1].join(".")
-          running_counts[ident] = Resque.redis.get(k).to_i
+          count_keys = Resque.redis.keys("concurrent.count.*")
+          running_counts = {}
+          count_keys.each do |k|
+            parts = k.split(".")
+            ident = parts[2..-1].join(".")
+            ident_sizes[ident] ||= {}
+            ident_sizes[ident]["running"] = Resque.redis.get(k).to_i
+          end
+
+          result[:identifiers] = ident_sizes
+        else
+          result[:identifiers] = {}
         end
 
+
         lock_keys = Resque.redis.keys("concurrent.lock.*")
-        lock_count = lock_keys.size
+        result[:lock_count] = lock_keys.size
 
         runnable_count = Resque.redis.scard(runnables_key)
+        result[:runnable_count] = runnable_count
 
-        return {
-            :queue_totals => {
-                :by_queue_name => queue_sizes,
-                :by_identifier => ident_sizes
-            },
-            :running_counts => running_counts,
-            :lock_count => lock_count,
-            :runnable_count => runnable_count,
-        }
+        return result
         
       end
 

data/lib/resque/plugins/concurrent_restriction/version.rb

@@ -1,7 +1,7 @@
 module Resque
   module Plugins
     module ConcurrentRestriction
-      VERSION = "0.5.1"
+      VERSION = "0.5.2"
     end
   end
 end

data/spec/concurrent_restriction_job_spec.rb

@@ -295,6 +295,25 @@ describe Resque::Plugins::ConcurrentRestriction do
       ConcurrentRestrictionJob.runnables("somequeue2").sort.should == []
     end
 
+    it "should track queue_counts" do
+      ConcurrentRestrictionJob.queue_counts.should == {}
+
+      job1 = Resque::Job.new("somequeue", {"class" => "ConcurrentRestrictionJob", "args" => [1]})
+      job2 = Resque::Job.new("somequeue", {"class" => "ConcurrentRestrictionJob", "args" => [2]})
+      job3 = Resque::Job.new("somequeue2", {"class" => "ConcurrentRestrictionJob", "args" => [3]})
+
+      ConcurrentRestrictionJob.push_to_restriction_queue(job1)
+      ConcurrentRestrictionJob.push_to_restriction_queue(job2)
+      ConcurrentRestrictionJob.push_to_restriction_queue(job3)
+      ConcurrentRestrictionJob.queue_counts.should == {"somequeue"=>2, "somequeue2"=>1}
+
+      ConcurrentRestrictionJob.pop_from_restriction_queue(ConcurrentRestrictionJob.tracking_key, "somequeue")
+      ConcurrentRestrictionJob.queue_counts.should == {"somequeue"=>1, "somequeue2"=>1}
+
+      ConcurrentRestrictionJob.pop_from_restriction_queue(ConcurrentRestrictionJob.tracking_key, "somequeue")
+      ConcurrentRestrictionJob.pop_from_restriction_queue(ConcurrentRestrictionJob.tracking_key, "somequeue2")
+      ConcurrentRestrictionJob.queue_counts.should == {"somequeue"=>0, "somequeue2"=>0}
+    end
   end
 
   context "#stash_if_restricted" do
@@ -456,6 +475,8 @@ describe Resque::Plugins::ConcurrentRestriction do
 
       IdentifiedRestrictionJob.runnables.sort.should == [ConcurrentRestrictionJob.tracking_key, IdentifiedRestrictionJob.tracking_key(1), IdentifiedRestrictionJob.tracking_key(2)]
 
+      ConcurrentRestrictionJob.queue_counts.should == {"queue1"=>2, "queue2"=>2, "queue3"=>1}
+
     end
 
   end
@@ -464,11 +485,21 @@ describe Resque::Plugins::ConcurrentRestriction do
     
     it "should have blank info when nothing going on" do
       stats = ConcurrentRestrictionJob.stats
-      stats[:queue_totals][:by_queue_name].should == {}
-      stats[:queue_totals][:by_identifier].should == {}
-      stats[:running_counts].should == {}
+      stats[:queues].should == {}
+      stats[:identifiers].should == {}
       stats[:lock_count].should == 0
       stats[:runnable_count].should == 0
+      estats = ConcurrentRestrictionJob.stats(true)
+      estats.should == stats
+    end
+
+    it "should not track identifiers when not extended" do
+      job1 = Resque::Job.new("queue1", {"class" => "IdentifiedRestrictionJob", "args" => [1]})
+      IdentifiedRestrictionJob.push_to_restriction_queue(job1)
+      IdentifiedRestrictionJob.set_running_count(IdentifiedRestrictionJob.tracking_key(1), 2)
+      
+      stats = IdentifiedRestrictionJob.stats
+      stats[:identifiers].should == {}
     end
 
     it "should track queue_totals" do
@@ -484,17 +515,18 @@ describe Resque::Plugins::ConcurrentRestriction do
       IdentifiedRestrictionJob.push_to_restriction_queue(job4)
       ConcurrentRestrictionJob.push_to_restriction_queue(job5)
 
-      stats = IdentifiedRestrictionJob.stats
-      stats[:queue_totals][:by_queue_name].should == {"queue1" => 2, "queue2" => 2, "queue3" => 1}
-      stats[:queue_totals][:by_identifier].should == {"IdentifiedRestrictionJob.1" => 2, "IdentifiedRestrictionJob.2" => 2, "ConcurrentRestrictionJob" => 1}
+      stats = IdentifiedRestrictionJob.stats(true)
+      stats[:queues].should == {"queue1" => 2, "queue2" => 2, "queue3" => 1}
+
+      stats[:identifiers].should == {"IdentifiedRestrictionJob.1"=>{"queue1"=>1, "queue2"=>1}, "IdentifiedRestrictionJob.2"=>{"queue1"=>1, "queue2"=>1}, "ConcurrentRestrictionJob"=>{"queue3"=>1}}
     end
 
     it "should track running_counts" do
       IdentifiedRestrictionJob.set_running_count(IdentifiedRestrictionJob.tracking_key(1), 2)
       IdentifiedRestrictionJob.set_running_count(IdentifiedRestrictionJob.tracking_key(2), 3)
       ConcurrentRestrictionJob.set_running_count(ConcurrentRestrictionJob.tracking_key, 4)
-      stats = IdentifiedRestrictionJob.stats
-      stats[:running_counts].should == {"IdentifiedRestrictionJob.1" => 2, "IdentifiedRestrictionJob.2" => 3, "ConcurrentRestrictionJob" => 4}
+      stats = IdentifiedRestrictionJob.stats(true)
+      stats[:identifiers].should == {"IdentifiedRestrictionJob.1"=>{"running"=>2}, "IdentifiedRestrictionJob.2"=>{"running"=>3}, "ConcurrentRestrictionJob"=>{"running"=>4}}
     end
 
     it "should track lock_count" do

metadata

@@ -2,7 +2,7 @@
 name: resque-concurrent-restriction
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors: 
 - Matt Conway
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-08 00:00:00 -04:00
+date: 2011-04-11 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency