brodock-work_queue 2.0.1

data/Gemfile

@@ -0,0 +1,12 @@
+source "http://rubygems.org"
+# Add dependencies required to use your gem here.
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "bundler", "~> 1.0.0"
+  gem "jeweler", "~> 1.6.4"
+  gem "rcov"
+end

data/Gemfile.lock

@@ -0,0 +1,20 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    git (1.2.5)
+    jeweler (1.6.4)
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      rake
+    rake (0.9.2)
+    rcov (0.9.10)
+    rcov (0.9.10-java)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0.0)
+  jeweler (~> 1.6.4)
+  rcov

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2009-2010 Miguel Fonseca
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,38 @@
+= Description
+
+A work queue is designed to coordinate work between a producer and a pool of worker threads.
+When some task needs to be performed, the producer adds an object containing the task routine to the work queue.
+If the work queue is full, the producer will block until a worker thread removes an object from the queue.
+Eventually, one of the worker threads removes the object from the work queue and executes the routine.
+If the work queue is empty, a worker thread will block until an object is made available by the producer.
+
+Work queues are useful for several reasons:
+* To easily perform tasks asynchronously and concurrently in your application;
+* To let you focus on the work you actually want to perform without having to worry about the thread creation and management;
+* To minimize overhead, by reusing previously constructed threads rather than creating new ones;
+* To bound resource use, by setting a limit on the maximum number of simultaneously executing threads;
+
+= Usage
+
+Install the gem:
+
+  gem install brodock-work_queue
+
+Or place this on your Gemfile:
+
+  gem "brodock-work_queue", :require => "work_queue" 
+
+Run the code:
+
+  require 'rubygems'
+  require 'work_queue'
+  wq = WorkQueue.new
+  wq.enqueue_b { puts "Hello from the WorkQueue" }
+  wq.join
+
+Note that you generally want to bound the resources used:
+
+  # Limit the maximum number of simultaneous worker threads
+  WorkQueue.new(10)
+  # Limit the maximum number of queued tasks
+  WorkQueue.new(nil,20)

data/Rakefile

@@ -0,0 +1,56 @@
+# encoding: utf-8
+$:.unshift File.dirname(__FILE__)
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+require 'lib/work_queue'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "brodock-work_queue"
+  gem.version = WorkQueue::VERSION
+  gem.homepage = "http://github.com/brodock/brodock-work_queue"
+  gem.license = "MIT"
+  gem.summary = %Q{A tunable work queue, designed to coordinate work between a producer and a pool of worker threads.}
+  gem.description = %Q{A tunable work queue, designed to coordinate work between a producer and a pool of worker threads.}
+  gem.email = "brodock@gmail.com"
+  gem.authors = ["Miguel Fonseca", "Gabriel Mazetto"]
+  gem.required_ruby_version = ">= 1.8.6"
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+require 'rcov/rcovtask'
+Rcov::RcovTask.new do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.rcov_opts << '--exclude "gems/*"'
+end
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "brodock-work_queue #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/lib/work_queue.rb

@@ -0,0 +1,236 @@
+##
+# = Name
+# WorkQueue
+#
+# == Description
+# This file contains an implementation of a work queue structure.
+#
+# == Version
+# 2.0.0
+#
+# == Author
+# Miguel Fonseca <fmmfonseca@gmail.com>
+#
+# == Copyright
+# Copyright 2009-2011 Miguel Fonseca
+#
+# == License
+# MIT (see LICENSE file)
+#
+
+require 'thread'
+require 'monitor'
+
+##
+# = WorkQueue
+#
+# == Description
+# A tunable work queue, designed to coordinate work between a producer and a pool of worker threads.
+#
+# == Usage
+#  wq = WorkQueue.new
+#  wq.enqueue_b { puts "Hello from the WorkQueue" }
+#  wq.join
+#
+class WorkQueue
+    
+    VERSION = "2.0.1"
+    
+    ##
+    # Creates a new work queue with the desired parameters.
+    #
+    #  wq = WorkQueue.new(5,10,20)
+    #
+    def initialize(max_threads=nil, max_tasks=nil)
+        self.max_threads = max_threads
+        self.max_tasks = max_tasks
+        @threads = Array.new
+        @threads.extend(MonitorMixin)
+        @threads_waiting = 0
+        @tasks = Array.new
+        @tasks.extend(MonitorMixin)
+        @task_enqueued = @tasks.new_cond
+        @task_completed = @tasks.new_cond
+        @cur_tasks = 0
+    end
+    
+    ##
+    # Returns the maximum number of worker threads.
+    # This value is set upon initialization and cannot be changed afterwards.
+    #
+    #  wq = WorkQueue.new()
+    #  wq.max_threads		#=> Infinity
+    #  wq = WorkQueue.new(1)
+    #  wq.max_threads		#=> 1
+    #
+    def max_threads
+        @max_threads
+    end
+    
+    ##
+    # Returns the current number of worker threads.
+    # This value is just a snapshot, and may change immediately upon returning.
+    #
+    #  wq = WorkQueue.new(10)
+    #  wq.cur_threads		#=> 0
+    #  wq.enqueue_b {}
+    #  wq.cur_threads		#=> 1
+    #
+    def cur_threads
+        @threads.size
+    end
+    
+    ##
+    # Returns the maximum number of queued tasks.
+    # This value is set upon initialization and cannot be changed afterwards.
+    #
+    #  wq = WorkQueue.new()
+    #  wq.max_tasks		#=> Infinity
+    #  wq = WorkQueue.new(nil,1)
+    #  wq.max_tasks		#=> 1
+    #
+    def max_tasks
+        @max_tasks
+    end
+    
+    ##
+    # Returns the current number of active tasks.
+    # This value is just a snapshot, and may change immediately upon returning.
+    #
+    #  wq = WorkQueue.new(1)
+    #  wq.enqueue_b { sleep(1) }
+    #  wq.cur_tasks		#=> 0
+    #  wq.enqueue_b {}
+    #  wq.cur_tasks		#=> 1
+    #
+    def cur_tasks
+        @cur_tasks
+    end
+    
+    ##
+    # Schedules the given Proc for future execution by a worker thread.
+    # If there is no space left in the queue, waits until space becomes available.
+    #
+    #  wq = WorkQueue.new(1)
+    #  wq.enqueue_p(Proc.new {})
+    #
+    def enqueue_p(proc, *args)
+        enqueue(proc, args) 
+    end
+    
+    ##
+    # Schedules the given Block for future execution by a worker thread.
+    # If there is no space left in the queue, waits until space becomes available.
+    #
+    #  wq = WorkQueue.new(1)
+    #  wq.enqueue_b {}
+    #
+    def enqueue_b(*args, &block)
+       enqueue(block, args) 
+    end
+    
+    ##
+    # Waits until the tasks queue is empty and all worker threads have finished.
+    #
+    #  wq = WorkQueue.new(1)
+    #  wq.enqueue_b { sleep(1) }
+    #  wq.join
+    #
+    def join
+        @tasks.synchronize do
+            @task_completed.wait_while { cur_tasks > 0 }
+        end
+    end
+    
+    ##
+    # Stops all worker threads immediately, aborting any ongoing tasks.
+    #
+    #  wq = WorkQueue.new(1)
+    #  wq.enqueue_b { sleep(1) }
+    #  wq.kill
+    #
+    def kill
+        @tasks.synchronize do
+            @threads.dup.each { |thread| thread.exit.join }
+            @threds.clear
+            @threads_waiting = 0
+            @tasks.clear
+            @cur_tasks = 0
+        end
+    end
+    
+    private
+    
+    ##
+    # Generic
+    #
+    def enqueue(proc, args)
+        @tasks.synchronize do
+            @task_completed.wait_while { cur_tasks >= max_tasks }
+            @tasks << [proc, args]
+            @cur_tasks += 1
+            @task_enqueued.signal
+            spawn_thread
+        end
+    end
+    
+    ##
+    # Sets the maximum number of worker threads.
+    #
+    def max_threads=(value)
+        raise ArgumentError, "the maximum number of threads must be positive" if value and value <= 0
+        @max_threads = value || 1.0/0
+    end
+    
+    ##
+    # Sets the maximum number of queued tasks.
+    #
+    def max_tasks=(value)
+        raise ArgumentError, "the maximum number of tasks must be positive" if value and value <= 0
+        @max_tasks = value || 1.0/0
+    end
+    
+    ##
+    # Enrolls a new worker thread.
+    # The request is only carried out if necessary.
+    #
+    def spawn_thread
+        @threads.synchronize do
+            if cur_threads < max_threads and @threads_waiting <= 0 and @tasks.size > 0
+                @threads << Thread.new do
+                    begin
+                        work
+                    ensure
+                        @threads.synchronize do
+                            @threads.delete(Thread.current)
+                        end
+                    end
+                end
+            end
+        end
+    end
+    
+    
+    ##
+    # Repeatedly process the tasks queue.
+    #
+    def work
+       loop do 
+            begin
+                proc, args = @tasks.synchronize do
+                    @threads_waiting += 1
+                    @task_enqueued.wait_while { @tasks.size <= 0 }
+                    @threads_waiting -= 1
+                    @tasks.shift
+                end
+                proc.call(*args)
+            ensure
+                @tasks.synchronize do
+                    @cur_tasks -= 1
+                    @task_completed.broadcast
+                end
+            end
+       end
+    end
+    
+end

data/test/helper.rb

@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'brodock-work_queue'
+
+class Test::Unit::TestCase
+end

data/test/test_work_queue.rb

@@ -0,0 +1,103 @@
+##
+# = Name
+# TC_WorkQueue
+#
+# == Description
+# This file contains unit tests for the WorkQueue class.
+#
+# == Author
+# Miguel Fonseca <fmmfonseca@gmail.com>
+#
+# == Copyright
+# Copyright 2009-2010 Miguel Fonseca
+#
+# == License
+# MIT (see LICENSE file)
+
+require 'test/unit'
+require 'lib/work_queue'
+
+class TestWorkQueue < Test::Unit::TestCase
+
+  # def setup
+  # end
+
+  # def teardown
+  # end
+
+  def test_enqueue
+    s = String.new
+    wq = WorkQueue.new
+    # using proc
+    wq.enqueue_p(Proc.new { |str| str.replace("Hello #1") }, s)
+    wq.join
+    assert_equal(s, "Hello #1")
+    # using block
+    wq.enqueue_b(s) { |str| str.replace("Hello #2") }
+    wq.join
+    assert_equal(s, "Hello #2")
+  end
+  
+  def test_inner_enqueue
+    s = String.new
+    wq = WorkQueue.new
+    wq.enqueue_b do
+      sleep 0.01
+      wq.enqueue_b(s) { |str| str.replace("Hello #1")  }
+      sleep 0.01
+    end
+    wq.join
+    assert_equal(s, "Hello #1")
+  end
+  
+  def test_threads_recycle
+    wq = WorkQueue.new
+    wq.enqueue_b { sleep 0.01 }
+    sleep 0.02
+    assert_equal(wq.cur_threads, 1)
+    wq.enqueue_b { sleep 0.01 }
+    assert_equal(wq.cur_threads, 1)
+    wq.join
+  end
+
+  def test_max_threads
+    assert_raise(ArgumentError) { WorkQueue.new(0) }
+    assert_raise(ArgumentError) { WorkQueue.new(-1) }
+    wq = WorkQueue.new(1)
+    assert_equal(wq.cur_threads, 0)
+    wq.enqueue_b { sleep(0.01) }
+    assert_equal(wq.cur_threads, 1)
+    wq.enqueue_b { sleep(0.01) }
+    assert_equal(wq.cur_threads, 1)
+    sleep(0.1)
+    assert_equal(wq.cur_threads, 1)
+    wq.join
+  end
+
+  def test_max_tasks
+    assert_raise(ArgumentError) { WorkQueue.new(nil,0) }
+    assert_raise(ArgumentError) { WorkQueue.new(nil,-1) }
+    wq = WorkQueue.new(1,1)
+    wq.enqueue_b { sleep(0.01) }
+    wq.enqueue_b { sleep(0.01) }
+    assert_equal(wq.cur_tasks, 1)
+    wq.enqueue_b { sleep(0.01) }
+    assert_equal(wq.cur_tasks, 1)
+    wq.join
+  end
+
+  def test_stress
+    a = []
+    m = Mutex.new
+    wq = WorkQueue.new(100,200)
+    (1..1000).each do
+      wq.enqueue_b(a,m) { |str,mut|
+        sleep(0.01)
+        mut.synchronize { a.push nil }
+      }
+    end
+    wq.join
+    assert_equal(a.size, 1000)
+  end
+
+end

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: brodock-work_queue
+version: !ruby/object:Gem::Version
+  version: 2.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Miguel Fonseca
+- Gabriel Mazetto
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-10-05 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: &2153041240 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *2153041240
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: &2153040760 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.4
+  type: :development
+  prerelease: false
+  version_requirements: *2153040760
+- !ruby/object:Gem::Dependency
+  name: rcov
+  requirement: &2153040280 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *2153040280
+description: A tunable work queue, designed to coordinate work between a producer
+  and a pool of worker threads.
+email: brodock@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE
+- README.rdoc
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.rdoc
+- Rakefile
+- lib/work_queue.rb
+- test/helper.rb
+- test/test_work_queue.rb
+homepage: http://github.com/brodock/brodock-work_queue
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.8.6
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: A tunable work queue, designed to coordinate work between a producer and
+  a pool of worker threads.
+test_files: []