work_queue 2.0.1 → 2.5.0

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2009-2010 Miguel Fonseca
+Copyright (c) 2012 Miguel Fonseca
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.rdoc

@@ -10,7 +10,8 @@ Work queues are useful for several reasons:
 * To easily perform tasks asynchronously and concurrently in your application;
 * To let you focus on the work you actually want to perform without having to worry about the thread creation and management;
 * To minimize overhead, by reusing previously constructed threads rather than creating new ones;
-* To bound resource use, by setting a limit on the maximum number of simultaneously executing threads;
+* To bound the resources used, by setting a limit on the maximum number of simultaneously executing threads;
+
 
 = Usage
 
@@ -26,9 +27,36 @@ Run the code:
   wq.enqueue_b { puts "Hello from the WorkQueue" }
   wq.join
 
-Note that you generally want to bound the resources used:
+It's generally recommended to bound the resources used:
 
   # Limit the maximum number of simultaneous worker threads
-  WorkQueue.new(10)
+  WorkQueue.new 10, nil
   # Limit the maximum number of queued tasks
-  WorkQueue.new(nil,20)
+  WorkQueue.new nil, 20
+
+
+= Example(s)
+
+Download multiple files in parallel: 
+
+  require 'open-uri'
+  require 'rubygems'
+  require 'work_queue'
+
+  wq = WorkQueue.new 5
+
+  (1..6605).each do |number|
+	  wq.enqueue_b do
+		  open("rfc#{number}.txt", "wb") do |file|
+		  file.write open("http://www.rfc-editor.org/rfc/rfc#{number}.txt").read
+			  puts "rfc#{number} downloaded"
+		  end
+	  end
+  end
+
+  wq.join
+
+
+= License
+
+Released under the MIT license.

data/lib/work_queue.rb

@@ -1,3 +1,6 @@
+require 'thread'
+require 'monitor'
+
 ##
 # = Name
 # WorkQueue
@@ -6,231 +9,259 @@
 # This file contains an implementation of a work queue structure.
 #
 # == Version
-# 2.0.0
+# 2.5.0
 #
 # == Author
-# Miguel Fonseca <fmmfonseca@gmail.com>
+# Miguel Fonseca <contact@miguelfonseca.com>
 #
 # == Copyright
-# Copyright 2009-2011 Miguel Fonseca
+# Copyright 2012 Miguel Fonseca
 #
 # == License
 # MIT (see LICENSE file)
 #
 
-require 'thread'
-require 'monitor'
-
-##
-# = WorkQueue
-#
-# == Description
-# A tunable work queue, designed to coordinate work between a producer and a pool of worker threads.
-#
-# == Usage
-#  wq = WorkQueue.new
-#  wq.enqueue_b { puts "Hello from the WorkQueue" }
-#  wq.join
-#
 class WorkQueue
-    
-    VERSION = "2.0.1"
-    
-    ##
-    # Creates a new work queue with the desired parameters.
-    #
-    #  wq = WorkQueue.new(5,10,20)
-    #
-    def initialize(max_threads=nil, max_tasks=nil)
-        self.max_threads = max_threads
-        self.max_tasks = max_tasks
-        @threads = Array.new
-        @threads.extend(MonitorMixin)
-        @threads_waiting = 0
-        @tasks = Array.new
-        @tasks.extend(MonitorMixin)
-        @task_enqueued = @tasks.new_cond
-        @task_completed = @tasks.new_cond
-        @cur_tasks = 0
-    end
-    
-    ##
-    # Returns the maximum number of worker threads.
-    # This value is set upon initialization and cannot be changed afterwards.
-    #
-    #  wq = WorkQueue.new()
-    #  wq.max_threads		#=> Infinity
-    #  wq = WorkQueue.new(1)
-    #  wq.max_threads		#=> 1
-    #
-    def max_threads
-        @max_threads
-    end
-    
-    ##
-    # Returns the current number of worker threads.
-    # This value is just a snapshot, and may change immediately upon returning.
-    #
-    #  wq = WorkQueue.new(10)
-    #  wq.cur_threads		#=> 0
-    #  wq.enqueue_b {}
-    #  wq.cur_threads		#=> 1
-    #
-    def cur_threads
-        @threads.size
-    end
-    
-    ##
-    # Returns the maximum number of queued tasks.
-    # This value is set upon initialization and cannot be changed afterwards.
-    #
-    #  wq = WorkQueue.new()
-    #  wq.max_tasks		#=> Infinity
-    #  wq = WorkQueue.new(nil,1)
-    #  wq.max_tasks		#=> 1
-    #
-    def max_tasks
-        @max_tasks
-    end
-    
-    ##
-    # Returns the current number of active tasks.
-    # This value is just a snapshot, and may change immediately upon returning.
-    #
-    #  wq = WorkQueue.new(1)
-    #  wq.enqueue_b { sleep(1) }
-    #  wq.cur_tasks		#=> 0
-    #  wq.enqueue_b {}
-    #  wq.cur_tasks		#=> 1
-    #
-    def cur_tasks
-        @cur_tasks
-    end
-    
-    ##
-    # Schedules the given Proc for future execution by a worker thread.
-    # If there is no space left in the queue, waits until space becomes available.
-    #
-    #  wq = WorkQueue.new(1)
-    #  wq.enqueue_p(Proc.new {})
-    #
-    def enqueue_p(proc, *args)
-        enqueue(proc, args) 
-    end
-    
-    ##
-    # Schedules the given Block for future execution by a worker thread.
-    # If there is no space left in the queue, waits until space becomes available.
-    #
-    #  wq = WorkQueue.new(1)
-    #  wq.enqueue_b {}
-    #
-    def enqueue_b(*args, &block)
-       enqueue(block, args) 
-    end
-    
-    ##
-    # Waits until the tasks queue is empty and all worker threads have finished.
-    #
-    #  wq = WorkQueue.new(1)
-    #  wq.enqueue_b { sleep(1) }
-    #  wq.join
-    #
-    def join
-        @tasks.synchronize do
-            @task_completed.wait_while { cur_tasks > 0 }
-        end
+  VERSION = "2.5.0"
+
+  ##
+  # Creates a new work queue with the desired parameters.
+  # It's generally recommended to bound the resources used.
+  #
+  # ==== Parameter(s)
+  # * +max_threads+ - Maximum number of worker threads.
+  # * +max_tasks+ - Maximum number of queued tasks.
+  #
+  # ==== Example(s)
+  #  wq = WorkQueue.new 10, nil
+  #  wq = WorkQueue.new nil, 20
+  #  wq = WorkQueue.new 10, 20
+  #
+  def initialize(max_threads=nil, max_tasks=nil)
+    self.max_threads = max_threads
+    self.max_tasks = max_tasks
+    @threads = Array.new
+    @threads_waiting = 0
+    @threads.extend MonitorMixin
+    @tasks = Array.new
+    @tasks.extend MonitorMixin
+    @task_enqueued = @tasks.new_cond
+    @task_dequeued = @tasks.new_cond
+    @task_completed = @tasks.new_cond
+    @tasks_pending = 0
+  end
+
+  ##
+  # Returns the maximum number of worker threads.
+  # This value is set upon initialization and cannot be changed afterwards.
+  #
+  # ==== Example(s)
+  #  wq = WorkQueue.new
+  #  wq.max_threads		#=> Infinity
+  #  wq = WorkQueue.new 1, nil
+  #  wq.max_threads		#=> 1
+  #
+  def max_threads
+    @max_threads
+  end
+
+  ##
+  # Returns the current number of worker threads.
+  # This value is just a snapshot, and may change immediately upon returning.
+  #
+  # ==== Example(s)
+  #  wq = WorkQueue.new 1
+  #  wq.cur_threads		#=> 0
+  #  wq.enqueue_b { ... }
+  #  wq.cur_threads		#=> 1
+  #
+  def cur_threads
+    @threads.size
+  end
+
+  ##
+  # Returns the maximum number of queued tasks.
+  # This value is set upon initialization and cannot be changed afterwards.
+  #
+  # ==== Example(s)
+  #  wq = WorkQueue.new
+  #  wq.max_tasks		#=> Infinity
+  #  wq = WorkQueue.new nil, 1
+  #  wq.max_tasks		#=> 1
+  #
+  def max_tasks
+    @max_tasks
+  end
+
+  ##
+  # Returns the current number of queued tasks.
+  # This value is just a snapshot, and may change immediately upon returning.
+  #
+  # ==== Example(s)
+  #  wq = WorkQueue.new 1
+  #  wq.enqueue_b { ... }
+  #  wq.cur_tasks		#=> 0
+  #  wq.enqueue_b { ... }
+  #  wq.cur_tasks		#=> 1
+  #
+  def cur_tasks
+    @tasks.size
+  end
+
+  ##
+  # Schedules the given Block for future execution by a worker thread.
+  # If there is no space left in the queue, waits until space becomes available.
+  #
+  # ==== Parameter(s)
+  # * +params+ - Parameters passed to the given block.
+  #
+  # ==== Example(s)
+  #  wq = WorkQueue.new
+  #  wq.enqueue_b("Parameter") { |obj| ... }
+  #
+  def enqueue_b(*params, &block)
+    enqueue block, params
+  end
+
+  ##
+  # Schedules the given Proc for future execution by a worker thread.
+  # If there is no space left in the queue, waits until space becomes available.
+  #
+  # ==== Parameter(s)
+  # * +proc+ - Proc to be executed.
+  # * +params+ - Parameters passed to the given proc.
+  #
+  # ==== Example(s)
+  #  wq = WorkQueue.new
+  #  wq.enqueue_p(Proc.new { |obj| ... }, "Parameter")
+  #
+  def enqueue_p(proc, *params)
+    enqueue proc, params
+  end
+
+  ##
+  # Waits until the tasks queue is empty and all worker threads have finished.
+  #
+  # ==== Example(s)
+  #  wq = WorkQueue.new
+  #  wq.enqueue_b { ... }
+  #  wq.join
+  #
+  def join
+    @tasks.synchronize do
+      @task_completed.wait_while { @tasks_pending > 0 }
     end
-    
-    ##
-    # Stops all worker threads immediately, aborting any ongoing tasks.
-    #
-    #  wq = WorkQueue.new(1)
-    #  wq.enqueue_b { sleep(1) }
-    #  wq.kill
-    #
-    def kill
-        @threads.dup.each { |thread| thread.exit.join }
+  end
+
+  ##
+  # Halt all worker threads immediately, aborting any ongoing tasks.
+  # Resets all 
+  #
+  # ==== Example(s)
+  #  wq = WorkQueue.new
+  #  wq.enqueue_b { ... }
+  #  wq.kill
+  #
+  def kill
+    @tasks.synchronize do
+      @threads.synchronize do
+        @threads.each { |thread| thread.exit }
         @threads.clear
         @threads_waiting = 0
-        @tasks.clear
-        @cur_tasks = 0
-    end
-    
-    private
-    
-    ##
-    # Generic
-    #
-    def enqueue(proc, args)
-        @tasks.synchronize do
-            @task_completed.wait_while { cur_tasks >= max_tasks }
-            @tasks << [proc, args]
-            @cur_tasks += 1
-            @task_enqueued.signal
-            spawn_thread
-        end
+      end
+      @tasks.clear
+      @tasks_pending = 0
+      @task_dequeued.broadcast
+      @task_completed.broadcast
     end
-    
-    ##
-    # Sets the maximum number of worker threads.
-    #
-    def max_threads=(value)
-        raise ArgumentError, "the maximum number of threads must be positive" if value and value <= 0
-        @max_threads = value || 1.0/0
+  end
+
+  private
+
+  ##
+  # Sets the maximum number of worker threads.
+  #
+  def max_threads=(value)
+    raise ArgumentError, "the maximum number of threads must be positive" if value and value <= 0
+    @max_threads = value || 1.0/0
+  end
+
+  ##
+  # Sets the maximum number of queued tasks.
+  #
+  def max_tasks=(value)
+    raise ArgumentError, "the maximum number of tasks must be positive" if value and value <= 0
+    @max_tasks = value || 1.0/0
+  end
+
+  ##
+  # Schedules the given Proc for future execution by a worker thread.
+  #
+  def enqueue(proc, params)
+    @tasks.synchronize do
+      @task_dequeued.wait_until { @tasks.size < @max_tasks }
+      @tasks << [proc, params]
+      @tasks_pending += 1
+      @task_enqueued.signal
     end
-    
-    ##
-    # Sets the maximum number of queued tasks.
-    #
-    def max_tasks=(value)
-        raise ArgumentError, "the maximum number of tasks must be positive" if value and value <= 0
-        @max_tasks = value || 1.0/0
+    spawn_thread
+  end
+
+  ##
+  # Enrolls a new worker thread. The request is only carried out if necessary.
+  #
+  def spawn_thread
+    @tasks.synchronize do
+      @threads.synchronize do
+        if @threads.size < @max_threads && @threads_waiting <= 0 && @tasks.size > 0
+          @threads << Thread.new { run }
+        end
+      end
     end
-    
-    ##
-    # Enrolls a new worker thread.
-    # The request is only carried out if necessary.
-    #
-    def spawn_thread
-        @threads.synchronize do
-            if cur_threads < max_threads and @threads_waiting <= 0 and @tasks.size > 0
-                @threads << Thread.new do
-                    begin
-                        work
-                    ensure
-                        @threads.synchronize do
-                            @threads.delete(Thread.current)
-                        end
-                    end
-                end
-            end
+  end
+
+  ##
+  # Repeatedly process the tasks queue.
+  #
+  def run
+    begin
+      loop do
+        proc, params = dequeue
+        begin
+          proc.call(*params)
+        rescue Exception => e
+          # Suppress Exception
         end
+        conclude
+      end
+    ensure
+      @threads.synchronize do
+        @threads.delete Thread.current
+      end
     end
-    
-    
-    ##
-    # Repeatedly process the tasks queue.
-    #
-    def work
-       loop do 
-            begin
-                proc, args = @tasks.synchronize do
-                    @threads_waiting += 1
-                    @task_enqueued.wait_while { @tasks.size <= 0 }
-                    @threads_waiting -= 1
-                    @tasks.shift
-                end
-                proc.call(*args)
-            rescue Exception => e
-                # Suppress Exception
-            ensure
-                @tasks.synchronize do
-                    @cur_tasks -= 1
-                    @task_completed.broadcast
-                end
-            end
-       end
+  end
+
+  ##
+  # Retrieves a task from the queue.
+  #
+  def dequeue
+    @tasks.synchronize do
+      @threads_waiting += 1
+      @task_enqueued.wait_while { @tasks.empty? }
+      @threads_waiting -= 1
+      task = @tasks.shift
+      @task_dequeued.signal
+      return task
+    end
+  end
+
+  ##
+  # Wind up a task execution.
+  #
+  def conclude
+    @tasks.synchronize do
+      @tasks_pending -= 1
+      @task_completed.signal
     end
-    
+  end
 end

data/tasks/test.rake

@@ -1,7 +1,6 @@
 require 'rake/testtask'
 
 namespace(:test) do
-
   # For a list of all attributes refer to http://rake.rubyforge.org/classes/Rake/TestTask.html
   Rake::TestTask.new(:unit) do |t|
     t.libs << "test"
@@ -9,5 +8,4 @@ namespace(:test) do
     t.verbose = true
     t.warning = true
   end
-
 end

data/test/tc_work_queue.rb

@@ -6,10 +6,10 @@
 # This file contains unit tests for the WorkQueue class.
 #
 # == Author
-# Miguel Fonseca <fmmfonseca@gmail.com>
+# Miguel Fonseca <contact@miguelfonseca.com>
 #
 # == Copyright
-# Copyright 2009-2010 Miguel Fonseca
+# Copyright 2012 Miguel Fonseca
 #
 # == License
 # MIT (see LICENSE file)
@@ -18,24 +18,20 @@ require 'test/unit'
 require 'lib/work_queue'
 
 class TC_WorkQueue < Test::Unit::TestCase
-
-  # def setup
-  # end
-
-  # def teardown
-  # end
-
-  def test_enqueue
+  def test_enqueue_proc
     s = String.new
     wq = WorkQueue.new
-    # using proc
-    wq.enqueue_p(Proc.new { |str| str.replace("Hello #1") }, s)
+    wq.enqueue_p(Proc.new { |str| str.replace "Hello Proc" }, s)
     wq.join
-    assert_equal(s, "Hello #1")
-    # using block
-    wq.enqueue_b(s) { |str| str.replace("Hello #2") }
+    assert_equal "Hello Proc", s
+  end
+  
+  def test_enqueue_block
+    s = String.new
+    wq = WorkQueue.new
+    wq.enqueue_b(s) { |str| str.replace "Hello Block" }
     wq.join
-    assert_equal(s, "Hello #2")
+    assert_equal "Hello Block", s
   end
   
   def test_inner_enqueue
@@ -43,69 +39,91 @@ class TC_WorkQueue < Test::Unit::TestCase
     wq = WorkQueue.new
     wq.enqueue_b do
       sleep 0.01
-      wq.enqueue_b(s) { |str| str.replace("Hello #1")  }
+      wq.enqueue_b(s) { |str| str.replace "Hello Inner" }
       sleep 0.01
     end
     wq.join
-    assert_equal(s, "Hello #1")
+    assert_equal "Hello Inner", s 
   end
   
   def test_threads_recycle
     wq = WorkQueue.new
     wq.enqueue_b { sleep 0.01 }
     sleep 0.02
-    assert_equal(wq.cur_threads, 1)
+    assert_equal 1, wq.cur_threads
     wq.enqueue_b { sleep 0.01 }
-    assert_equal(wq.cur_threads, 1)
+    assert_equal 1, wq.cur_threads
     wq.join
   end
 
   def test_max_threads
-    assert_raise(ArgumentError) { WorkQueue.new(0) }
-    assert_raise(ArgumentError) { WorkQueue.new(-1) }
-    wq = WorkQueue.new(1)
-    assert_equal(wq.cur_threads, 0)
-    wq.enqueue_b { sleep(0.01) }
-    assert_equal(wq.cur_threads, 1)
-    wq.enqueue_b { sleep(0.01) }
-    assert_equal(wq.cur_threads, 1)
-    sleep(0.1)
-    assert_equal(wq.cur_threads, 1)
+    wq = WorkQueue.new 1
+    assert_equal 0, wq.cur_threads
+    wq.enqueue_b { sleep 0.01 }
+    assert_equal 1, wq.cur_threads
+    wq.enqueue_b { sleep 0.01 }
+    assert_equal 1, wq.cur_threads
+    sleep 0.1
+    assert_equal 1, wq.cur_threads
     wq.join
   end
+  
+  def test_max_threads_validation
+    assert_raise(ArgumentError) { WorkQueue.new 0, nil }
+    assert_raise(ArgumentError) { WorkQueue.new -1, nil }
+  end
 
   def test_max_tasks
-    assert_raise(ArgumentError) { WorkQueue.new(nil,0) }
-    assert_raise(ArgumentError) { WorkQueue.new(nil,-1) }
-    wq = WorkQueue.new(1,1)
-    wq.enqueue_b { sleep(0.01) }
-    wq.enqueue_b { sleep(0.01) }
-    assert_equal(wq.cur_tasks, 1)
-    wq.enqueue_b { sleep(0.01) }
-    assert_equal(wq.cur_tasks, 1)
+    wq = WorkQueue.new 1, 1
+    wq.enqueue_b { sleep 0.01 }
+    wq.enqueue_b { sleep 0.01 }
+    assert_equal 1, wq.cur_tasks
+    wq.enqueue_b { sleep 0.01 }
+    assert_equal 1, wq.cur_tasks
     wq.join
   end
+  
+  def test_max_tasks_validation
+    assert_raise(ArgumentError) { WorkQueue.new nil, 0 }
+    assert_raise(ArgumentError) { WorkQueue.new nil, -1 }
+  end
 
   def test_stress
-    a = []
+    i = 0
     m = Mutex.new
-    wq = WorkQueue.new(100,200)
-    (1..1000).each do
-      wq.enqueue_b(a,m) { |str,mut|
-        sleep(0.01)
-        mut.synchronize { a.push nil }
+    wq = WorkQueue.new 100, 200
+    (1..10000).each do
+      wq.enqueue_b {
+        sleep 0.01
+        m.synchronize { i += 1 }
       }
     end
     wq.join
-    assert_equal(a.size, 1000)
+    assert_equal 10000, i
+  end
+  
+  def test_stress_prolonged
+    i = 0
+    m = Mutex.new
+    wq = WorkQueue.new 100, 200
+    (1..10000).each do
+      wq.enqueue_b {
+        sleep rand(5)
+        m.synchronize { i += 1 }
+      }
+    end
+    wq.join
+    assert_equal 10000, i
   end
 
   def test_kill
     s = String.new
     wq = WorkQueue.new
-    wq.enqueue_b(s) { |str| sleep 0.1; str.replace("Hello")  }
+    wq.enqueue_b(s) { |str| 
+      sleep 0.1
+      str.replace "Hello"
+    }
     wq.kill
-    assert_equal(s, "")
+    assert(s.empty?)
   end
-
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: work_queue
 version: !ruby/object:Gem::Version 
-  hash: 13
+  hash: 27
   prerelease: 
   segments: 
   - 2
+  - 5
   - 0
-  - 1
-  version: 2.0.1
+  version: 2.5.0
 platform: ruby
 authors: 
 - Miguel Fonseca
@@ -15,11 +15,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-11-21 00:00:00 Z
+date: 2012-04-19 00:00:00 Z
 dependencies: []
 
 description: 
-email: fmmfonseca@gmail.com
+email: contact@miguelfonseca.com
 executables: []
 
 extensions: []
@@ -50,12 +50,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 59
+      hash: 57
       segments: 
       - 1
       - 8
-      - 6
-      version: 1.8.6
+      - 7
+      version: 1.8.7
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
@@ -68,7 +68,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.8.11
+rubygems_version: 1.8.22
 signing_key: 
 specification_version: 3
 summary: A tunable work queue, designed to coordinate work between a producer and a pool of worker threads.