work_queue 0.1.0

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2009 Miguel Fonseca
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,11 @@
+Conceptually, a work queue is designed to coordinate work between a producer and a set of worker threads.
+When some task needs to be performed, the producer adds an object containing the task routine to the work queue.
+If the work queue is full, the producer will block until a worker thread removes an object from the queue.
+Eventually, one of the worker threads removes the object from the work queue and executes the routine.
+If the work queue is empty, a worker thread will block until an object is made available by the producer or until the keep-alive time expires.
+
+Work queues are useful for several reasons:
+* To easily perform tasks asynchronously and concurrently in your application;
+* To let you focus on the work you actually want to perform without having to worry about the thread creation and management;
+* To minimize overhead, by reusing previously constructed threads rather than creating new ones;
+* To bound resource use, by setting a limit on the maximum number of simultaneously executing threads;

data/Rakefile

@@ -0,0 +1,9 @@
+require 'rubygems'
+require 'rake'
+require 'rake/clean'
+
+# Load all rakefile extensions
+Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].each { |ext| load ext }
+
+# Set default task
+task :default => ["test:unit"]

data/lib/work_queue.rb

@@ -0,0 +1,237 @@
+##
+# = Name
+# WorkQueue
+#
+# == Description
+# This file contains an implementation of a work queue structure.
+#
+# == Version
+# 0.1.0
+#
+# == Author
+# Miguel Fonseca <fmmfonseca@gmail.com>
+#
+# == Copyright
+# Copyright 2009 Miguel Fonseca
+#
+# == License
+# MIT (see LICENSE file)
+#
+
+require 'thread'
+require 'timeout'
+
+##
+# = WorkQueue
+#
+# == Description
+# A tunable work queue, designed to coordinate work between a producer and a set of worker threads.
+#
+# == Usage
+#  wq = WorkQueue.new(1)
+#  wq.enqueue_b { puts "Hello from the WorkQueue" }
+#  wq.join
+#
+class WorkQueue
+	
+	VERSION = "0.1.0"
+	
+	##
+	# Creates a new work queue.
+	#	
+	#  wq = WorkQueue.new(5,10,20)
+	#
+	def initialize(max_threads=nil, max_tasks=nil, keep_alive=60)
+		self.max_threads = max_threads
+		self.max_tasks = max_tasks
+		self.keep_alive = keep_alive
+		@threads = []
+		@threads_lock = Mutex.new
+		@tasks = max_tasks ? SizedQueue.new(max_tasks) : Queue.new
+		@threads.taint
+		@tasks.taint
+		self.taint
+	end
+	
+	##
+	# Returns the maximum number of worker threads.
+	# This value is set upon initialization and cannot be changed afterwards.
+	#
+	#  wq = WorkQueue.new(1)
+	#  wq.max_threads		#=> 1
+	#
+	def max_threads
+		@max_threads
+	end
+	
+	##
+	# Returns the current number of worker threads.
+	# This value is just a snapshot, and may change immediately upon returning.
+	#
+	#  wq = WorkQueue.new(10)
+	#  wq.cur_threads		#=> 0
+	#  wq.enqueue_b {}
+	#  wq.cur_threads		#=> 1
+	#
+	def cur_threads
+		@threads.size
+	end
+	
+	##
+	# Returns the maximum number of queued tasks.
+	# This value is set upon initialization and cannot be changed afterwards.
+	#
+	#  wq = WorkQueue.new(nil,1)
+	#  wq.max_tasks		#=> 1
+	#
+	def max_tasks
+		@max_tasks
+	end
+	
+	##
+	# Returns the current number of queued tasks.
+	# This value is just a snapshot, and may change immediately upon returning.
+	#
+	#  wq = WorkQueue.new(1)
+	#  wq.enqueue_b { sleep(1) }
+	#  wq.cur_tasks		#=> 0
+	#  wq.enqueue_b {}
+	#  wq.cur_tasks		#=> 1
+	#
+	def cur_tasks
+		@tasks.size
+	end
+	
+	##
+	# Returns the number of seconds to keep worker threads alive waiting for new tasks.
+	# This value is set upon initialization and cannot be changed afterwards.
+	#
+	#  wq = WorkQueue.new()
+	#  wq.keep_alive		#=> 60
+	#  wq = WorkQueue.new(nil,nil,1)
+	#  wq.keep_alive		#=> 1
+	#
+	def keep_alive
+		@keep_alive
+	end
+	
+	##
+	# Schedules the given Proc for execution by a worker thread.
+	# If there is no space left in the queue, waits until space becomes available.
+	#
+	#  wq = WorkQueue.new(1)
+	#  wq.enqueue_p(Proc.new {})
+	#
+	def enqueue_p(proc, *args)
+		@tasks << [proc,args]
+		spawn_thread
+		self
+	end
+	
+	##
+	# Schedules the given Block for execution by a worker thread.
+	# If there is no space left in the queue, waits until space becomes available.
+	#
+	#  wq = WorkQueue.new(1)
+	#  wq.enqueue_b {}
+	#
+	def enqueue_b(*args, &block)
+		@tasks << [block,args]
+		spawn_thread
+		self
+	end
+	
+	##
+	# Waits until the tasks queue is empty.
+	#
+	#  wq = WorkQueue.new(1)
+	#  wq.enqueue_b { sleep(1) }
+	#  wq.join
+	#
+	def join
+		cur_threads.times { dismiss_thread }
+		@threads.dup.each { |t| t.join }
+		self
+	end
+	
+	##
+	# Stops all worker threads immediately, aborting any ongoing tasks.
+	#
+	#  wq = WorkQueue.new(1)
+	#  wq.enqueue_b { sleep(1) }
+	#  wq.stop
+	#
+	def stop
+		@threads.dup.each { |t| t.exit.join }
+		@tasks.clear
+		self
+	end
+	
+	private
+	
+	##
+	# Sets the maximum number of worker threads.
+	#
+	def max_threads=(value)
+		raise ArgumentError, "the maximum number of threads must be positive" if value and value <= 0
+		@max_threads = value || 1.0/0
+	end
+	
+	##
+	# Sets the maximum number of queued tasks.
+	#
+	def max_tasks=(value)
+		raise ArgumentError, "the maximum number of tasks must be positive" if value and value <= 0
+		@max_tasks = value || 1.0/0
+	end
+	
+	##
+	# Sets the maximum time to keep worker threads alive waiting for new tasks.
+	#
+	def keep_alive=(value)
+		raise ArgumentError, "the keep-alive time must be positive" if value and value <= 0
+		@keep_alive = value || 1.0/0
+	end
+	
+	##
+	# Spawns a new worker thread.
+	#
+	def spawn_thread
+		if cur_threads < max_threads and @tasks.num_waiting <= 0
+			@threads_lock.synchronize { 
+				@threads << Thread.new do
+					begin
+						work()
+					ensure
+						@threads_lock.synchronize { @threads.delete(Thread.current) }
+					end
+				end
+			}
+		end
+	end
+	
+	##
+	# Dismisses an existing worker thread.
+	#
+	def dismiss_thread
+		@tasks << [Proc.new { Thread.exit }, nil] if cur_threads > 0
+	end
+	
+	##
+	# Repeatedly process the tasks queue.
+	#
+	def work
+		loop do
+			begin
+				proc, args = timeout(keep_alive) { @tasks.pop }
+				proc.call(*args)
+			rescue Timeout::Error
+				break
+			rescue Exception
+				# suppress exception
+			end
+			break if cur_threads > max_threads
+		end
+	end
+	
+end

data/tasks/gem.rake

@@ -0,0 +1,26 @@
+require 'rake/gempackagetask'
+require 'lib/work_queue'
+
+CLEAN.include("pkg")
+
+# For a list of all attributes refer to http://docs.rubygems.org/read/chapter/20
+spec = Gem::Specification.new do |s|
+	s.name = "work_queue"
+	s.version = WorkQueue::VERSION
+	s.summary = "A tunable work queue, designed to coordinate work between a producer and a set of worker threads."
+	s.homepage = "http://github.com/fmmfonseca/work_queue"
+	s.author = "Miguel Fonseca"
+	s.email = "fmmfonseca@gmail.com"
+	
+	s.required_ruby_version = ">= 1.8.6"
+	s.files = FileList["LICENSE", "Rakefile", "README.rdoc", "tasks/*.rake", "lib/**/*.rb", "test/tc_*.rb"].to_a
+	s.test_files = Dir.glob("test/tc_*.rb")
+	s.has_rdoc = true
+	s.extra_rdoc_files = ["README.rdoc"]
+	s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "WorkQueue", "--main", "README.rdoc"]
+end
+
+# For a list of all attributes refer to http://rake.rubyforge.org/classes/Rake/PackageTask.html
+Rake::GemPackageTask.new(spec) do |p|
+	p.need_zip = true
+end

data/tasks/rdoc.rake

@@ -0,0 +1,10 @@
+require 'rake/rdoctask'
+
+CLEAN.include("doc")
+
+# For a list of all attributes refer to http://rake.rubyforge.org/classes/Rake/RDocTask.html
+Rake::RDocTask.new do |rd|
+	rd.main = "README.rdoc"
+	rd.rdoc_files.include("README.rdoc", "lib/**/*.rb")
+	rd.rdoc_dir = "doc"
+end

data/tasks/test.rake

@@ -0,0 +1,22 @@
+require 'rake/testtask'
+
+namespace(:test) do
+
+	# For a list of all attributes refer to http://rake.rubyforge.org/classes/Rake/TestTask.html
+	Rake::TestTask.new(:unit) do |t|
+		t.libs << "test"
+		t.test_files = FileList['test/tc_*.rb']
+		t.verbose = true
+		t.warning = true
+	end
+
+	desc "Run tests on multiple ruby versions"
+	task(:compatibility) do
+		versions = %w[1.8.6 1.8.7 1.9.1 jruby]
+		system <<-CMD
+			bash -c 'source ~/.rvm/scripts/rvm;
+					 rvm #{versions.join(',')} rake test:unit'
+		CMD
+	end
+
+end

data/test/tc_work_queue.rb

@@ -0,0 +1,89 @@
+##
+# = Name
+# TC_WorkQueue
+#
+# == Description
+# This file contains unit tests for the WorkQueue class.
+#
+# == Author
+# Miguel Fonseca <fmmfonseca@gmail.com>
+#
+# == Copyright
+# Copyright 2009 Miguel Fonseca
+#
+# == License
+# MIT (see LICENSE file)
+
+require 'test/unit'
+require 'lib/work_queue'
+
+class TC_WorkQueue < Test::Unit::TestCase
+	
+	# def setup
+	# end
+	
+	# def teardown
+	# end
+	
+	def test_enqueue
+		s = String.new
+		wq = WorkQueue.new
+		# Using Proc
+		wq.enqueue_p(Proc.new { |str| str.replace("Hello #1") }, s)
+		wq.join
+		assert_equal(s, "Hello #1")
+		# Using Block
+		wq.enqueue_b(s) { |str| str.replace("Hello #2") }
+		wq.join
+		assert_equal(s, "Hello #2")
+	end
+	
+	def test_max_threads
+		assert_raise(ArgumentError) { WorkQueue.new(0) }
+		assert_raise(ArgumentError) { WorkQueue.new(-1) }
+		wq = WorkQueue.new(1)
+		assert_equal(wq.cur_threads, 0)
+		wq.enqueue_b { sleep(0.01) }
+		assert_equal(wq.cur_threads, 1)
+		wq.enqueue_b { sleep(0.01) }
+		assert_equal(wq.cur_threads, 1)
+		sleep(0.1)
+		assert_equal(wq.cur_threads, 1)
+		wq.join
+	end
+	
+	def test_max_tasks
+		assert_raise(ArgumentError) { WorkQueue.new(nil,0) }
+		assert_raise(ArgumentError) { WorkQueue.new(nil,-1) }
+		wq = WorkQueue.new(1,1)
+		wq.enqueue_b { sleep(0.01) }
+		wq.enqueue_b { sleep(0.01) }
+		assert_equal(wq.cur_tasks, 1)
+		wq.enqueue_b { sleep(0.01) }
+		assert_equal(wq.cur_tasks, 1)
+		wq.join
+	end
+	
+	def test_keep_alive
+		assert_raise(ArgumentError) { WorkQueue.new(nil,nil,0) }
+		assert_raise(ArgumentError) { WorkQueue.new(nil,nil,-1) }
+		wq = WorkQueue.new(1,1,0.01)
+		wq.enqueue_b { sleep(0.01) }
+		assert_equal(wq.cur_threads, 1)
+		sleep(0.1)
+		assert_equal(wq.cur_threads, 0)
+		wq.join
+	end
+	
+	def test_stress
+		a = []
+		m = Mutex.new
+		wq = WorkQueue.new(100)
+		(1..1000).each {
+			wq.enqueue_b(a,m) { |str,mut| sleep(0.01); mut.synchronize { a.push nil } }
+		}
+		wq.join
+		assert_equal(a.size, 1000)
+	end
+	
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification 
+name: work_queue
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Miguel Fonseca
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-11-22 00:00:00 +00:00
+default_executable: 
+dependencies: []
+
+description: 
+email: fmmfonseca@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- LICENSE
+- Rakefile
+- README.rdoc
+- tasks/gem.rake
+- tasks/rdoc.rake
+- tasks/test.rake
+- lib/work_queue.rb
+- test/tc_work_queue.rb
+has_rdoc: true
+homepage: http://github.com/fmmfonseca/work_queue
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --line-numbers
+- --inline-source
+- --title
+- WorkQueue
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: 1.8.6
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A tunable work queue, designed to coordinate work between a producer and a set of worker threads.
+test_files: 
+- test/tc_work_queue.rb