throttle-queue 0.0.1 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 602c620f6b090b5696a141185d5e1d65c8240c07
-  data.tar.gz: 852eec61c6d3d3c1200e83b63a202d03d57a4842
+  metadata.gz: 0d0f4a538c9d23d2a21d79b63122c4a0074e4095
+  data.tar.gz: 16859c8f40a47777526ec299ed4721ba955ca55f
 SHA512:
-  metadata.gz: 53b009d91bafd808374a49e10a98e19fd926f98bee250e163f77bf9da7ffad395528ec33df9855e57b75f338dd3579c7ce5234b1a6954df8769470700511a6e3
-  data.tar.gz: a430b7bb92bde90fe9bf937198b2337171bf98292d4ae72fbc3924998b827d9298c33020c3fa939bffaad021107533992110aebc8c11fbe8ca15e39e42f7cfe3
+  metadata.gz: a39194f269635e9b8ece84714010df1e1cedd0194c5133a77fdfb277d71227aa4bf88e1d57e4b5fba2c84e470831a04b5ede2d8f8f287dd83baac54154f29229
+  data.tar.gz: dc984657a36e8238294813d3f10e36ae837bd77764ee0c6a8c2be81648cedaeb2dda1a5334658a75ffab4cf01435ab6e02fe1f24284b73cb74c9029345c7b0ba

data/LICENSE.txt

@@ -19,3 +19,4 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
+

data/README.md

@@ -18,10 +18,24 @@ Or install it yourself as:
 
     $ gem install throttle-queue
 
+## Purpose
+
+Sometimes server resources are rate-limited. If you as a client exceed the server's
+limit, you become temporarily black-listed. A popular set of free APIs, which rhymes
+with Foogle Maps, has strict limits in place after which you start seeing 403s instead
+of 200s.
+
+Let's say, for the above reason or another, you want to accumulate a cache of objects
+no faster than N times a second. You're happy to startup your app quickly and let your
+cache grow in the background. When a user requests a resource, you can bump it to the
+front of your queue and block until it is ready.
+
 ## Usage
 
 Create a queue and add background work
 
+	require 'throttle-queue'
+
 	q = ThrottleQueue.new 3
 	files.each {|file|
 		q.background(file) {
@@ -39,6 +53,39 @@ Wait for everything to finish
 
 	q.wait
 
+## Details
+
+Each resource is assumed to have a unique identifier, e.g. a filename or a reproducible
+hash value. The queue does not check if the resource exists first, but it will check if
+the id has already been queued. Any time an id is added to the queue, the previous block
+is dropped in favor of the new.
+
+Once an id has made it through the queue and been processed, the same id can be added
+again and will be blindly processed again. It is assumed the user of the object knows
+whether the resource already exists, and will decide whether or not a given id should be
+added.
+
+## Multiple Processes
+
+Just bring in the multi-process-aware queue wrapper
+
+	require 'throttle-queue/multi-process'
+
+	q = ThrottleQueue::MultiProcess.new 3
+	files.each {|file|
+		q.background(file) {
+			fetch file
+		}
+	}
+
+Use this queue the same way in each process. Make sure each queue id has the same
+meaning in each process (e.g. file paths should be absolute or relative to the same
+working directory). If your processes race to stack up a bunch of background work, the
+first process to add each work will be the only one to execute its block.
+
+Do not fork after a multi-process queue is created. If your process forks itself,
+create each queue after the fork.
+
 ## Contributing
 
 1. Fork it
@@ -46,3 +93,4 @@ Wait for everything to finish
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create new Pull Request
+

data/Rakefile

@@ -6,3 +6,4 @@ Rake::TestTask.new {|t|
 
 desc 'Run tests'
 task :default => :test
+

data/lib/throttle-queue.rb

@@ -1,222 +1 @@
-require 'thread'
-# ThrottleQueue is a thread-safe rate-limited work queue. It allows both
-# background and foreground operations.
-#
-# Example:
-#   q = ThrottleQueue 3
-#   files.each {|file|
-#     q.background(file) {|id|
-#       fetch file
-#     }
-#   }
-class ThrottleQueue
-	# Creates a new ThrottleQueue with the given rate limit (per second).
-	def initialize(limit)
-		raise "refusing to do zero work per second" if limit <= 0
-		@limit = limit
-
-		@queue = PriorityQueue.new
-
-		@mutex = Mutex.new
-		@pausing = ConditionVariable.new
-		@idle = ConditionVariable.new
-		@in_flight = nil
-		@processing_thread = nil
-		@items = {}
-
-		@throttling = nil
-		@state = :idle
-		@t0 = Time.now
-	end
-	# Signals the queue to stop processing and shutdown.
-	#
-	# Items still in the queue are dropped. Any item
-	# currently in flight will finish.
-	def shutdown
-		@queue.shutdown
-		@pausing.signal
-	end
-	# Returns true if there is nothing queued and no
-	# threads are running
-	def idle?
-		@state == :idle
-	end
-	# Blocks the calling thread while the queue processes work.
-	#
-	# Returns after the timeout has expired, or after the
-	# queue returns to the idle state.
-	def wait(timeout = nil)
-		@mutex.synchronize {
-			@idle.wait(@mutex, timeout) unless idle?
-		}
-	end
-	# Adds work to the queue to run in the background, and
-	# returns immediately.
-	#
-	# If the block takes an argument, it will be passed the
-	# same id used to queue the work.
-	def background(id, &block)
-		@mutex.synchronize {
-			if id != @in_flight
-				@items[id] = block
-				@queue.background id
-				run
-			end
-		}
-	end
-	# Adds work to the queue ahead of all background work, and
-	# blocks until the given block has been called.
-	#
-	# Will preempt an id of the same value in either the
-	# background or foreground queues.
-	#
-	# If the block takes an argument, it will be passed the
-	# same id used to queue the work.
-	def foreground(id, &block)
-		t = nil
-		@mutex.synchronize {
-			if id == @in_flight
-				t = @processing_thread unless @processing_thread == Thread.current
-			else
-				b = @items[id]
-				b.kill if b.is_a? FG
-
-				t = @items[id] = FG.new block, self
-
-				@queue.foreground id
-				run
-			end
-		}
-		t.join if t
-	end
-
-	private
-	def run
-		return unless @state == :idle
-		@state = :running
-		@throttling = Thread.new {
-			loop {
-				break if @queue.shutdown? or @queue.empty?
-
-				elapsed = Time.now - @t0
-				wait_time = 1.0 / @limit + 0.01
-				if @processing_thread and elapsed < wait_time
-					@mutex.synchronize {
-						@pausing.wait @mutex, wait_time - elapsed
-					}
-				end
-
-				if id = @queue.pop
-					@mutex.synchronize {
-						@in_flight = id
-						@processing_thread = Thread.new {
-							block = @items[@in_flight]
-							if block.arity == 0
-								block.call
-							else
-								block.call @in_flight
-							end
-						}
-					}
-					@processing_thread.join if @processing_thread
-				end
-
-				@t0 = Time.now
-			}
-
-			@mutex.synchronize {
-				@state = :idle
-				if @queue.shutdown? or @queue.empty?
-					@idle.signal
-				else
-					# Restart to prevent a join deadlock
-					send :run
-				end
-			}
-		}
-	end
-	class FG #:nodoc: all
-		def initialize(block, h)
-			@block = block
-			@thread = Thread.new {
-				Thread.stop
-				@block.call *@args
-			}
-			@h = h
-		end
-		def arity
-			@block.arity
-		end
-		def call(*args)
-			@args = args
-			@thread.run
-		end
-		def kill
-			@thread.kill
-		end
-		def join
-			@thread.join
-		end
-	end
-	class PriorityQueue #:nodoc: all
-		def initialize
-			@mutex = Mutex.new
-			@fg = []
-			@bg = []
-			@received = ConditionVariable.new
-			@shutdown = false
-		end
-
-		def shutdown
-			@shutdown = true
-			@received.signal
-		end
-
-		def shutdown?
-			@shutdown
-		end
-
-		def empty?
-			@mutex.synchronize {
-				@fg.empty? and @bg.empty?
-			}
-		end
-
-		def background(id)
-			@mutex.synchronize {
-				unless @shutdown || @bg.include?(id)
-					@bg << id
-					@received.signal
-				end
-			}
-		end
-
-		def foreground(id)
-			@mutex.synchronize {
-				unless @shutdown || @fg.include?(id)
-					@fg << id
-					if @bg.include?(id)
-						@bg.delete id
-					else
-						@received.signal
-					end
-				end
-			}
-		end
-
-		def pop
-			@mutex.synchronize {
-				if @fg.empty? and @bg.empty?
-					@received.wait(@mutex) unless @shutdown
-				end
-
-				if @shutdown
-				elsif ! @fg.empty?
-					@fg.shift
-				else
-					@bg.shift
-				end
-			}
-		end
-	end
-end
+require_relative 'throttle-queue/single-process'

data/lib/throttle-queue/multi-process.rb

@@ -0,0 +1,103 @@
+require 'drb'
+require 'fileutils'
+require_relative 'single-process.rb'
+
+class ThrottleQueue
+	# ThrottleQueue::MultiProcess is a wrapper around ThrottleQueue
+	# that shares the queue between multiple processes.
+	#
+	# Example:
+	#   q = ThrottleQueue::MultiProcess 3
+	#   files.each {|file|
+	#     q.background(file) {|id|
+	#       fetch file
+	#     }
+	#   }
+	class MultiProcess
+		# Creates a new ThrottleQueue::MultiProcess with the given rate limit (per second).
+		#
+		# If this is the first instace of the shared queue, it becomes the master queue and
+		# starts a DRbServer instace. If a DRbServer is already running, it connects to the
+		# queue as a remote DRbObject.
+		def initialize(limit, name = 'ThrottleQueue')
+			tmp = "/tmp/#{name}.sock"
+			FileUtils.touch tmp
+			File.open(tmp, 'r+') {|f|
+				f.flock File::LOCK_EX
+				begin
+					port = f.read.to_i
+					if port == 0
+						@queue = ThrottleQueue.new(limit)
+						@drb = DRb.start_service nil, @queue
+						f.seek 0, IO::SEEK_SET
+						f.truncate 0
+						f.write @drb.uri[/\d+$/]
+						f.flock File::LOCK_UN
+					else
+						@queue = DRbObject.new_with_uri("druby://localhost:#{port}")
+						@queue.idle?
+						@drb = DRb.start_service
+						f.flock File::LOCK_UN
+					end
+				rescue DRb::DRbConnError
+					f.seek 0, IO::SEEK_SET
+					f.truncate 0
+					retry
+				end
+			}
+
+		end
+		# Signals the queue to stop processing and shutdown.
+		#
+		# The DRbServer is shutdown in either the master process or any
+		# client process.
+		def shutdown
+			@queue.shutdown
+			@drb.stop_service if @drb
+		end
+		# Returns true if there is nothing queued and no
+		# threads are running
+		def idle?
+			@queue.idle?
+		end
+		# Blocks the calling thread while the queue processes work.
+		#
+		# Returns after the timeout has expired, or after the
+		# queue returns to the idle state.
+		def wait(timeout = nil)
+			begin
+				@queue.wait(timeout)
+			rescue DRb::DRbConnError
+			end
+		end
+		# Adds work to the queue to run in the background, and
+		# returns immediately.
+		#
+		# If the block takes an argument, it will be passed the
+		# same id used to queue the work.
+		#
+		# The block may be preempted by a foreground job started in
+		# this or another process. If not preempted, the block will
+		# run in this process.
+		def background(id, &block)
+			@queue.background(id, &block)
+		end
+		# Adds work to the queue ahead of all background work, and
+		# blocks until the given block has been called.
+		#
+		# Will preempt an id of the same value in the
+		# background queue, and wait on an id of the same value already
+		# in the foreground queue.
+		#
+		# If the block takes an argument, it will be passed the
+		# same id used to queue the work.
+		#
+		# The block may wait on an already queued foreground job in
+		# this or another process. If so queued, this block will not
+		# run. If the block does run, it will run in this process.
+		def foreground(id, &block)
+			@queue.foreground(id, &block)
+		end
+	end
+end
+

data/lib/throttle-queue/single-process.rb

@@ -0,0 +1,224 @@
+require 'thread'
+# ThrottleQueue is a thread-safe rate-limited work queue. It allows both
+# background and foreground operations.
+#
+# Example:
+#   q = ThrottleQueue 3
+#   files.each {|file|
+#     q.background(file) {|id|
+#       fetch file
+#     }
+#   }
+class ThrottleQueue
+	# Creates a new ThrottleQueue with the given rate limit (per second).
+	def initialize(limit)
+		raise "refusing to do zero work per second" if limit <= 0
+		@limit = limit
+
+		@queue = PriorityQueue.new
+
+		@mutex = Mutex.new
+		@pausing = ConditionVariable.new
+		@idle = ConditionVariable.new
+		@in_flight = nil
+		@processing_thread = nil
+		@items = {}
+
+		@throttling = nil
+		@state = :idle
+		@t0 = Time.now
+	end
+	# Signals the queue to stop processing and shutdown.
+	#
+	# Items still in the queue are dropped. Any item
+	# currently in flight will finish.
+	def shutdown
+		@queue.shutdown
+		@pausing.signal
+	end
+	# Returns true if there is nothing queued and no
+	# threads are running
+	def idle?
+		@state == :idle
+	end
+	# Blocks the calling thread while the queue processes work.
+	#
+	# Returns after the timeout has expired, or after the
+	# queue returns to the idle state.
+	def wait(timeout = nil)
+		@mutex.synchronize {
+			@idle.wait(@mutex, timeout) unless idle?
+		}
+	end
+	# Adds work to the queue to run in the background, and
+	# returns immediately.
+	#
+	# If the block takes an argument, it will be passed the
+	# same id used to queue the work.
+	def background(id, &block)
+		@mutex.synchronize {
+			unless @items.has_key? id
+				@items[id] = block
+				@queue.background id
+				run
+			end
+		}
+	end
+	# Adds work to the queue ahead of all background work, and
+	# blocks until the given block has been called.
+	#
+	# Will preempt an id of the same value in the
+	# background queue, and wait on an id of the same value already
+	# in the foreground queue.
+	#
+	# If the block takes an argument, it will be passed the
+	# same id used to queue the work.
+	def foreground(id, &block)
+		t = nil
+		@mutex.synchronize {
+			if id == @in_flight
+				t = @processing_thread unless @processing_thread == Thread.current
+			else
+				t = @items[id]
+				unless t.is_a? FG
+					t = @items[id] = FG.new block, self
+					@queue.foreground id
+					run
+				end
+			end
+		}
+		t.join if t
+	end
+
+	private
+	def run
+		return unless @state == :idle
+		@state = :running
+		@throttling = Thread.new {
+			loop {
+				break if @queue.shutdown? or @queue.empty?
+
+				elapsed = Time.now - @t0
+				wait_time = 1.0 / @limit + 0.01
+				if @processing_thread and elapsed < wait_time
+					@mutex.synchronize {
+						@pausing.wait @mutex, wait_time - elapsed
+					}
+				end
+
+				if id = @queue.pop
+					@mutex.synchronize {
+						@in_flight = id
+						@processing_thread = Thread.new {
+							block = @items[@in_flight]
+							if block.arity == 0
+								block.call
+							else
+								block.call @in_flight
+							end
+						}
+					}
+					@processing_thread.join if @processing_thread
+
+					@mutex.synchronize {
+						@items.delete @in_flight
+						@in_flight = nil
+					}
+				end
+
+				@t0 = Time.now
+			}
+
+			@mutex.synchronize {
+				@state = :idle
+				if @queue.shutdown? or @queue.empty?
+					@idle.broadcast
+				else
+					# Restart to prevent a join deadlock
+					send :run
+				end
+			}
+		}
+	end
+	class FG #:nodoc: all
+		def initialize(block, h)
+			@block = block
+			@thread = Thread.new {
+				Thread.stop unless @args
+				@block.call *@args
+			}
+			@h = h
+		end
+		def arity
+			@block.arity
+		end
+		def call(*args)
+			@args = args
+			@thread.run
+		end
+		def join
+			@thread.join
+		end
+	end
+	class PriorityQueue #:nodoc: all
+		def initialize
+			@mutex = Mutex.new
+			@fg = []
+			@bg = []
+			@received = ConditionVariable.new
+			@shutdown = false
+		end
+
+		def shutdown
+			@shutdown = true
+			@received.signal
+		end
+
+		def shutdown?
+			@shutdown
+		end
+
+		def empty?
+			@mutex.synchronize {
+				@fg.empty? and @bg.empty?
+			}
+		end
+
+		def background(id)
+			@mutex.synchronize {
+				unless @shutdown || @bg.include?(id)
+					@bg << id
+					@received.signal
+				end
+			}
+		end
+
+		def foreground(id)
+			@mutex.synchronize {
+				unless @shutdown || @fg.include?(id)
+					@fg << id
+					if @bg.include?(id)
+						@bg.delete id
+					else
+						@received.signal
+					end
+				end
+			}
+		end
+
+		def pop
+			@mutex.synchronize {
+				if @fg.empty? and @bg.empty?
+					@received.wait(@mutex) unless @shutdown
+				end
+
+				if @shutdown
+				elsif ! @fg.empty?
+					@fg.shift
+				else
+					@bg.shift
+				end
+			}
+		end
+	end
+end

data/test/multiprocess-test.rb

@@ -0,0 +1,88 @@
+require_relative '../lib/throttle-queue/multi-process'
+require 'test/unit'
+require 'thread'
+
+class ThrottleQueueMultiProcessTest < Test::Unit::TestCase
+
+	def testSingleProcess
+		t = ThrottleQueue::MultiProcess.new 10
+
+		results = []
+		%w(apple banana cake donut egg).each {|w|
+			t.background(w) {
+				results << w.capitalize
+			}
+		}
+		t.wait
+		assert_equal %w(Apple Banana Cake Donut Egg), results
+	ensure
+		t.shutdown
+	end
+
+	def testTwoProcesses
+		p = fork {
+			t = ThrottleQueue::MultiProcess.new 10
+			%w(fig grape ham ice jelly).each {|w|
+				t.background(w) {
+					File.open('results.txt', 'a') {|f|
+						f.puts w.capitalize
+					}
+				}
+			}
+			t.wait
+		}
+
+		t = ThrottleQueue::MultiProcess.new 10
+
+		results = []
+		%w(apple banana cake donut egg).each {|w|
+			t.background(w) {
+				results << w.capitalize
+			}
+		}
+		t.wait
+		assert_equal %w(Apple Banana Cake Donut Egg), results
+		assert_equal %w(Fig Grape Ham Ice Jelly), File.open('results.txt') {|f| f.readlines.map &:chomp}
+	ensure
+		t.shutdown
+		FileUtils.rm_f 'results.txt'
+	end
+
+	def testTwoProcessesWithFG
+		rd, wr = IO.pipe
+
+		p = fork {
+			rd.close
+			t = ThrottleQueue::MultiProcess.new 10
+			%w(fig grape ham ice jelly).each {|w|
+				t.background(w) {
+					File.open('results.txt', 'a') {|f|
+						f.puts w.capitalize
+						if w == 'grape'
+							wr.close
+						end
+					}
+				}
+			}
+			t.wait
+		}
+
+		wr.close
+
+		t = ThrottleQueue::MultiProcess.new 10
+		rd.read
+		rd.close
+
+		t.foreground('apple') {|w|
+			File.open('results.txt', 'a') {|f|
+				f.puts w.capitalize
+			}
+		}
+
+		t.wait
+		assert_equal %w(Fig Grape Apple Ham Ice Jelly), File.open('results.txt') {|f| f.readlines.map &:chomp}
+	ensure
+		t.shutdown
+		FileUtils.rm_f 'results.txt'
+	end
+end

data/test/throttle-queue-test.rb

@@ -1,5 +1,6 @@
 require_relative '../lib/throttle-queue'
 require 'test/unit'
+require 'thread'
 
 class ThrottleQueueTest < Test::Unit::TestCase
 
@@ -61,7 +62,7 @@ class ThrottleQueueTest < Test::Unit::TestCase
 		}
 
 		@t.wait
-		assert_equal %w(Apple Banana CAKEYO Donut Egg), results
+		assert_equal %w(Apple Banana Cake Donut Egg), results
 	end
 	def testForegroundInFlight
 		results = []
@@ -88,7 +89,6 @@ class ThrottleQueueTest < Test::Unit::TestCase
 				results << 'CAKEYO'
 			}
 		}
-
 		%w(apple banana cake donut egg).each {|w|
 			@t.background(w) {
 				results << w.capitalize
@@ -124,6 +124,32 @@ class ThrottleQueueTest < Test::Unit::TestCase
 		@t.wait
 		assert_equal %w(Apple Banana Fish Grape Cake Donut Egg), results
 	end
+	def testForegroundWaitOnQueuedForeground
+		results = []
+		threads = []
+
+		ids = Queue.new
+		ids << 'apple' << 'banana' << 'banana' << 'banana'
+
+		values = Queue.new
+		values << 'Apple' << 'Banana' << 'BANANAYO' << 'DUDE'
+
+		ids.size.times {
+			threads << Thread.new {
+				@t.foreground(ids.pop) {
+					results << values.pop
+				}
+			}
+		}
+
+		threads.each {|t|
+			t.join
+		}
+		@t.wait
+		assert_equal %w(Apple Banana), results
+		assert_equal 0, ids.size
+		assert_equal 2, values.size
+	end
 	def testShutdownWithoutWaiting
 		results = []
 		%w(apple banana cake donut egg).each {|w|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: throttle-queue
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.0
 platform: ruby
 authors:
 - Ryan Calhoun
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-20 00:00:00.000000000 Z
+date: 2014-12-09 00:00:00.000000000 Z
 dependencies: []
 description: A thread-safe rate-limited work queue, which allows for background and
   foreground operations.
@@ -22,6 +22,9 @@ files:
 - README.md
 - Rakefile
 - lib/throttle-queue.rb
+- lib/throttle-queue/multi-process.rb
+- lib/throttle-queue/single-process.rb
+- test/multiprocess-test.rb
 - test/throttle-queue-test.rb
 homepage: https://github.com/theryan/throttle-queue
 licenses:
@@ -49,4 +52,5 @@ specification_version: 4
 summary: A thread-safe rate-limited work queue
 test_files:
 - test/throttle-queue-test.rb
+- test/multiprocess-test.rb
 - Rakefile