queue_bundle 0.0.1

data/lib/queue_bundle/version.rb

@@ -0,0 +1,3 @@
+class QueueBundle
+  VERSION = "0.0.1"
+end

data/lib/queue_bundle.rb

@@ -0,0 +1,214 @@
+require "queue_bundle/version"
+
+require 'thread'
+
+class QueueNotFound < StandardError; end
+
+class QueueBundle
+  
+
+  
+  attr_reader :hash_algorithm
+  def initialize(size, options = {})
+    @mutex = Mutex.new
+    @queues = []
+    @size = size
+    @closed = false
+    @hash_algorithm = options[:hash_algorithm] || :simple_hash
+    @key_lookup = options[:key_lookup] || :id
+    1.upto(@size) {|i|
+      @queues << Queue.new
+    }
+  end
+  def is_queue?
+    true
+  end
+  def simple_hash(key)
+    key.hash % @size
+  end
+  def simple_index(key)
+    key
+  end
+  
+ # def resize(new_size)
+ #   
+ # end
+
+  def closed?
+    @closed
+  end
+  def empty?(index = nil)
+    self.length(index) == 0
+  end
+  def reassign(index)
+    @mutex.synchronize do
+      dest_queue = next_queue(index)
+      if !dest_queue.nil?
+        old_queue = @queues[index]
+        @queues[index] = :reassigned
+        if !old_queue.empty?
+          begin
+            while !old_queue.empty?
+              dest_queue.push(old_queue.pop(true))
+            end
+          rescue ThreadError
+            
+          end
+          
+        end
+        true
+      else
+        false
+      end
+    end
+  end
+
+  def close()
+    @closed = true
+  end
+  def push(obj, key = nil)
+    key ||= get_key(obj)
+    queue = get_queue(key)
+    
+    if queue.nil?
+      raise QueueNotFound.new("Queue not found for #{key}")
+    else
+      queue.push(obj)
+    end
+  end
+  alias_method :enq, :push
+  def pop(index, non_block = false)
+    queue = @queues[index]
+    
+    if queue.nil? 
+      raise QueueNotFound.new("Queue not found for #{key}")
+    elsif queue == :reassigned
+      raise QueueNotFound.new("Queue reassigned")
+    else
+      queue.pop(non_block)
+    end
+  end
+  alias_method :deq, :pop
+  def clear(index = nil)
+    if index.nil?
+      active_queues.each {|queue| queue.clear}
+    else
+      queue = @queues[index]   #doesn't switch to next queue so that it will error out
+      
+      if queue.nil? 
+        raise QueueNotFound.new("Queue not found for #{key}")
+      elsif queue == :reassigned
+        raise QueueNotFound.new("Queue reassigned")
+      else
+        queue.clear()
+      end
+    end
+  end
+  def num_waiting(index = nil)
+    if index.nil?
+      active_queues.map {|queue| queue.num_waiting}.inject{|sum,x| sum + x }
+    else
+      queue = @queues[index]
+      
+      if queue.nil? 
+        raise QueueNotFound.new("Queue not found for #{key}")
+      elsif queue == :reassigned
+        raise QueueNotFound.new("Queue reassigned")
+      else
+        queue.num_waiting
+      end
+    end
+  end
+  def empty?(index = nil)
+    if index.nil?
+      active_queues.inject{|total,y|  (!total || !y) ? false : true }  # if it sees a false then it is not empty
+    else
+      queue = @queues[index]
+      
+      if queue.nil? 
+        raise QueueNotFound.new("Queue not found for #{key}")
+      elsif queue == :reassigned
+        raise QueueNotFound.new("Queue reassigned")
+      else
+        queue.empty?
+      end
+    end
+  end
+  
+  def length(index = nil)
+    if index.nil?
+      active_queues.map {|queue| queue.length}.inject{|sum,x| sum + x }
+    else
+      queue = @queues[index]
+      
+      if queue.nil? 
+        raise QueueNotFound.new("Queue not found for #{key}")
+      elsif queue == :reassigned
+        raise QueueNotFound.new("Queue reassigned")
+      else
+        queue.length
+      end
+    end
+  end
+  
+  alias_method :size, :length
+  
+  def lengths
+    length_hash = {}
+    active_queues.each_index {|queue_index| 
+      length_hash[queue_index] = @queues[queue_index].length
+    }
+    length_hash
+  end
+  alias_method :sizes, :lengths
+  private
+  
+    def get_key(obj)
+      key = nil
+      if @key_lookup.kind_of?(Proc)
+        key = @key_lookup.call obj
+      elsif @key_lookup.kind_of?(Symbol)
+        if obj.kind_of?(Hash)
+          key = obj[@key_lookup]
+        else
+          key = obj.send(@key_lookup)
+        end
+      end
+      key
+    end
+    def get_queue(key)
+      queue_index = nil
+      if @hash_algorithm.kind_of?(Proc)
+        queue_index = @hash_algorithm.call key
+      elsif @hash_algorithm.kind_of?(Symbol)
+        queue_index = self.send(@hash_algorithm, key)
+      end
+      queue = @queues[queue_index]
+      if queue == :reassigned
+        queue = next_queue(queue_index)
+      end
+      
+      queue
+    end
+    def active_queues
+      @queues.select {|queue| queue != :reassigned }
+    end
+    def next_queue(index)
+      current_index = index
+      while n = next_index(current_index) 
+        break if n == index
+        if @queues[n] != :reassigned
+          return @queues[n]
+        end
+        current_index = n
+      end
+      nil
+    end
+    def next_index(index)
+      n = index + 1
+      if n >= @size
+        n = 0
+      end
+      n
+    end
+end

data/test/helper.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'queue_bundle'
+
+class Test::Unit::TestCase
+end

data/test/test_queue_bundle.rb

@@ -0,0 +1,236 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'helper'
+require 'ostruct'
+
+class TestQueueBundle < Test::Unit::TestCase
+
+  def test_simple_index
+    queue = QueueBundle.new(5, :hash_algorithm => :simple_index)
+    assert_equal :simple_index, queue.hash_algorithm
+    assert_raise(QueueNotFound) do
+       queue.push(1, 50)
+    end
+    0.upto(4) {|i|
+      queue.push(i, i)
+    }
+    assert_equal 5, queue.length
+    0.upto(4) {|i|
+      assert_equal 1, queue.length(i)
+    }
+    0.upto(4) {|i|
+      assert_equal i, queue.pop(i) 
+      assert_equal 0, queue.length(i)
+      assert_raise(ThreadError) do
+        queue.pop(i, true)
+      end
+      
+    }
+    assert_equal 0, queue.length
+    assert_equal false, queue.closed?
+    queue.close
+    assert_equal true, queue.closed?
+  end
+  
+  def test_simple_hash
+    queue = QueueBundle.new(5, :hash_algorithm => :simple_hash)
+    assert_equal :simple_hash, queue.hash_algorithm
+   
+    0.upto(4) {|i|
+      queue.push(i, i)
+    }
+    
+    assert_equal 5, queue.length
+    results = []
+    0.upto(4) {|i|
+      1.upto(queue.length(i)) {
+        results << queue.pop(i)
+      }
+    }
+    assert_equal [0,1,2,3,4], results.sort
+    assert_equal 0, queue.length
+    assert_equal false, queue.closed?
+    queue.close
+    assert_equal true, queue.closed?
+  end
+  
+  def test_simple_index_with_hash_key
+    queue = QueueBundle.new(5, :hash_algorithm => :simple_index)
+    assert_equal :simple_index, queue.hash_algorithm
+    assert_raise(QueueNotFound) do
+       queue.push(1, 50)
+    end
+    0.upto(4) {|i|
+      queue.push({:id => i})
+    }
+    assert_equal 5, queue.length
+    0.upto(4) {|i|
+      assert_equal 1, queue.length(i)
+    }
+    0.upto(4) {|i|
+      assert_equal i, queue.pop(i)[:id]
+      assert_equal 0, queue.length(i)
+      assert_raise(ThreadError) do
+        queue.pop(i, true)
+      end
+      
+    }
+    assert_equal 0, queue.length
+    assert_equal false, queue.closed?
+    queue.close
+    assert_equal true, queue.closed?
+  end
+  
+  def test_simple_index_with_hash_alt_key
+    queue = QueueBundle.new(5, :hash_algorithm => :simple_index, :key_lookup => :alt)
+    assert_equal :simple_index, queue.hash_algorithm
+    assert_raise(QueueNotFound) do
+       queue.push(1, 50)
+    end
+    0.upto(4) {|i|
+      queue.push({:alt => i})
+    }
+    assert_equal 5, queue.length
+    0.upto(4) {|i|
+      assert_equal 1, queue.length(i)
+    }
+    0.upto(4) {|i|
+      assert_equal i, queue.pop(i)[:alt]
+      assert_equal 0, queue.length(i)
+      assert_raise(ThreadError) do
+        queue.pop(i, true)
+      end
+      
+    }
+    assert_equal 0, queue.length
+    assert_equal false, queue.closed?
+    queue.close
+    assert_equal true, queue.closed?
+  end
+
+  def test_simple_index_with_obj_alt_key
+    queue = QueueBundle.new(5, :hash_algorithm => :simple_index, :key_lookup => :alt)
+    assert_equal :simple_index, queue.hash_algorithm
+    assert_raise(QueueNotFound) do
+       queue.push(1, 50)
+    end
+    0.upto(4) {|i|
+      queue.push(OpenStruct.new({:alt => i}))
+    }
+    assert_equal 5, queue.length
+    0.upto(4) {|i|
+      assert_equal 1, queue.length(i)
+    }
+    0.upto(4) {|i|
+      assert_equal i, queue.pop(i).alt
+      assert_equal 0, queue.length(i)
+      assert_raise(ThreadError) do
+        queue.pop(i, true)
+      end
+      
+    }
+    assert_equal 0, queue.length
+    assert_equal false, queue.closed?
+    queue.close
+    assert_equal true, queue.closed?
+  end
+
+  def test_simple_index_with_reassign
+    queue = QueueBundle.new(5, :hash_algorithm => :simple_index)
+    assert_equal :simple_index, queue.hash_algorithm
+    assert_raise(QueueNotFound) do
+       queue.push(1, 50)
+    end
+    0.upto(4) {|i|
+      queue.push(i, i)
+    }
+    assert_equal 5, queue.length
+    0.upto(4) {|i|
+      assert_equal 1, queue.length(i)
+    }
+    assert_equal true, queue.reassign(4)
+    assert_equal 2, queue.length(0)  #wrapped
+    1.upto(3) {|i|
+      assert_equal 1, queue.length(i)
+    }
+    assert_raise(QueueNotFound) do
+        queue.pop(4, true)
+      end
+    assert_equal 0, queue.pop(0) 
+    assert_equal 4, queue.pop(0) 
+    
+    1.upto(3) {|i|
+      assert_equal i, queue.pop(i) 
+      assert_equal 0, queue.length(i)
+      assert_raise(ThreadError) do
+        queue.pop(i, true)
+      end
+      
+    }
+    
+    assert_equal 0, queue.length
+    queue.push(4,4)
+    assert_equal 1, queue.length(0)
+    
+    assert_equal false, queue.closed?
+    queue.close
+    assert_equal true, queue.closed?
+    
+    
+  end
+  
+  def test_simple_index_with_reassign_all
+    queue = QueueBundle.new(5, :hash_algorithm => :simple_index)
+    assert_equal :simple_index, queue.hash_algorithm
+    assert_raise(QueueNotFound) do
+       queue.push(1, 50)
+    end
+    0.upto(4) {|i|
+      queue.push(i, i)
+    }
+    assert_equal true, queue.reassign(3)
+    assert_equal 2, queue.size(4)
+    assert_equal true, queue.reassign(4)
+    assert_equal 3, queue.size(0)
+    assert_equal true, queue.reassign(0)
+    assert_equal 4, queue.size(1)
+    assert_equal true, queue.reassign(1)
+    assert_equal 5, queue.size(2)
+    assert_equal false, queue.reassign(2)
+
+    
+    assert_equal 2, queue.pop(2)
+    assert_equal 1, queue.pop(2)
+    assert_equal 0, queue.pop(2)
+    assert_equal 4, queue.pop(2)
+    assert_equal 3, queue.pop(2)
+
+ 
+  end
+  def test_doc_example
+    queue = QueueBundle.new(5, :key_lookup => :id)
+    threads = []
+    0.upto(4).each {|worker_id|
+      threads << Thread.new do 
+        loop do
+          break if queue.closed? && queue.empty?(worker_id)
+          work = queue.pop(worker_id)   #in this case this would be blocking
+        end
+      end
+    }
+    0.upto(50).each {|work_id|
+      queue.push({id: work_id, name: "task"})
+    }
+    
+    queue.close
+    
+    threads.compact.each do |t|
+      begin
+        t.join
+      rescue Interrupt
+      end
+    end
+  end
+  
+end
+

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification 
+name: queue_bundle
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Pete Brumm
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-12 00:00:00 Z
+dependencies: []
+
+description: Allows queue work to be distributed to seperate threads that need a consistent end point.   Allows you to provide a hashing algorithm for which thread handles the work.
+email: 
+- pbrumm@edgenet.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/queue_bundle.rb
+- lib/queue_bundle/version.rb
+- test/helper.rb
+- test/test_queue_bundle.rb
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: queue_bundle
+rubygems_version: 1.7.2
+signing_key: 
+specification_version: 3
+summary: Provides a way to split queue put's into multiple seperate queue pulls
+test_files: 
+- test/helper.rb
+- test/test_queue_bundle.rb