parallel_queue 0.1.0 → 0.1.1

data/README.md

@@ -18,7 +18,14 @@ Example:
     require 'redis'
     require 'parallel_queue'
     redis = Redis.new(:host => '127.0.0.1', :port => '6379')
-    queue = ParallelQueue.new(redis, 'my_object_message_queue')
+    queue = ParallelQueue.new(redis, 'my_object_message_queue', :maxlength => 1000)
+    # The optional :maxlength option limits the length of each individual message queue.
+    # When a queue that already has maxlength messages receives a new message, the
+    # oldest message in that queue is discarded (O(1)).
+    # Because there is one queue per message emitter, one queue becoming full has no effect
+    # on the remaining queues (no messages will be lost for other emitters unless they,
+    # too, reach :maxlength). If maxlength is omitted, then queue length is not
+    # artificially limited.
     queue.enqueue('123', 'hello world')
     queue.enqueue('peanuts', 'Chalie Brown')
     queue.enqueue('peanuts', 'Snoopy')
@@ -67,13 +74,15 @@ Terminal 2:
 Terminal 3 (run the command in terminal 3 after starting 1 & 2):
 
     $ ruby enqueue_demo.rb
-    After a few seconds have passed, press control-c in Terminal 3.
+    This will automatically end after enqueueing 40000 times
     Terminals 1 & 2 will automatically stop when they finish processing
     the data enqueued by Terminal 3.
 
 To check for any common values between d1.txt and d2.txt (there should be none):
 
     $ comm -1 -2 d1.txt d2.txt
+    should not have any matches (make sure you delete
+    the two files between runs)
 
 
 

data/Rakefile

@@ -18,7 +18,7 @@ Jeweler::Tasks.new do |gem|
   gem.homepage = "http://github.com/populr/parallel_queue"
   gem.license = "MIT"
   gem.summary = %Q{A thread safe, Redis backed, parallel queue abstraction}
-  gem.description = %Q{Motivation for creating this was for queueing messages based on ID so that very chatty message emitters don't prevent messages from others from being processed}
+  gem.description = %Q{A thread safe, Redis backed, parallel queue abstraction. Motivation for creating this was for queueing messages based on ID so that very chatty message emitters don't prevent messages from others from being processed}
   gem.email = "daniel@populr.me"
   gem.authors = ["Daniel Nelson"]
   # dependencies defined in Gemfile

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/dequeue_demo.rb

@@ -13,11 +13,13 @@ filename = ARGV[0]
 return puts "File name required" if filename.nil?
 
 array = []
+start = nil
 while array.empty? || queue.queue_count > 0
+  start = Time.now if start.nil? && !array.empty?
   queue.dequeue_each() do |item|
     array << item.to_i
     puts "dequeue: #{item}"
   end
 end
-
+puts Time.now - start
 File.open(filename, 'w') { |f| f.write(array.sort.join("\n")) }

data/enqueue_demo.rb

@@ -7,11 +7,11 @@ require 'pry-nav'
 require 'pry-stack_explorer'
 
 redis = Redis.new(:host => '127.0.0.1', :port => '6379')
-queue = ParallelQueue.new(redis, 'demo_queue')
+queue = ParallelQueue.new(redis, 'demo_queue', :maxlength => 1000)
 
 counter = 0
 ids = ['abc', 'def', 'ghi', 'jkl', 'mno', 'pqrs']
-while true
+while counter < 40000
   id = ids.sample
   queue.enqueue(id, counter)
   puts "enqueue: #{counter}"

data/lib/parallel_queue.rb

@@ -45,6 +45,27 @@ class ParallelQueue
       release_lock
       item
     else # couldn't acquire or break the lock. wait and try again
+      # A small sleep value is actually faster than no sleep value, presumably because no
+      # delay puts too much stress on Redis
+      # Experiment:
+      # started dequeue_demo.rb in two terminals
+      # started enqueue_demo.rb in a third terminal
+      # enqueue_demo.rb is set to run until it enqueues 40,000 times
+      # dequeue_demo.rb times its run, starting from the first time there is data to
+      # dequeue, and running through until all data have been dequeued
+      # times reported are the average of both dequeue terminals (which consitently ended
+      # within 0.1 second of one another)
+      # Sleep Delay            Run Duration (in seconds)
+      # 0.01                   22.6
+      # 0.001                  22.3
+      # no sleep               25.1
+      # 0.001                  23.8
+      # 0.001                  23.8
+      # 0.01                   22.6
+      # 0.01                   22.7
+      # no sleep               25.1
+      # no sleep               25.0
+      #
       sleep 0.01
       dequeue
     end
@@ -69,6 +90,8 @@ class ParallelQueue
       self.current_queue_index = 0
       release_lock
     else # couldn't acquire or break the lock. wait and try again
+      # a small sleep value is actually faster than no sleep value, presumably because no
+      # delay puts too much stress on Redis
       sleep 0.01
       delete_all!
     end
@@ -95,8 +118,10 @@ class ParallelQueue
   end
 
   def current_queue_index # :nodoc:
-    return 0 if queue_count == 0
-    @current_queue_index % queue_count
+    # in a multi-threaded environment, queue_count can go from non-zero to zero
+    # between a conditional check and the mod computation, so rescue the exception
+    # rather than trying to prevent it
+    @current_queue_index % queue_count rescue 0
   end
 
   def current_queue_index=(index) # :nodoc:

data/parallel_queue.gemspec

@@ -5,12 +5,12 @@
 
 Gem::Specification.new do |s|
   s.name = "parallel_queue"
-  s.version = "0.1.0"
+  s.version = "0.1.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Daniel Nelson"]
-  s.date = "2012-05-11"
-  s.description = "Motivation for creating this was for queueing messages based on ID so that very chatty message emitters don't prevent messages from others from being processed"
+  s.date = "2012-05-14"
+  s.description = "A thread safe, Redis backed, parallel queue abstraction. Motivation for creating this was for queueing messages based on ID so that very chatty message emitters don't prevent messages from others from being processed"
   s.email = "daniel@populr.me"
   s.extra_rdoc_files = [
     "LICENSE",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: parallel_queue
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-11 00:00:00.000000000 Z
+date: 2012-05-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &2161456900 !ruby/object:Gem::Requirement
+  requirement: &2161227500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: 2.2.0
   type: :runtime
   prerelease: false
-  version_requirements: *2161456900
+  version_requirements: *2161227500
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2161489940 !ruby/object:Gem::Requirement
+  requirement: &2161261120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2161489940
+  version_requirements: *2161261120
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &2161488860 !ruby/object:Gem::Requirement
+  requirement: &2161260240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2161488860
+  version_requirements: *2161260240
 - !ruby/object:Gem::Dependency
   name: jeweler
-  requirement: &2161488220 !ruby/object:Gem::Requirement
+  requirement: &2161259560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2161488220
+  version_requirements: *2161259560
 - !ruby/object:Gem::Dependency
   name: pry
-  requirement: &2161487480 !ruby/object:Gem::Requirement
+  requirement: &2161258840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2161487480
+  version_requirements: *2161258840
 - !ruby/object:Gem::Dependency
   name: pry-nav
-  requirement: &2161486340 !ruby/object:Gem::Requirement
+  requirement: &2161257440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2161486340
+  version_requirements: *2161257440
 - !ruby/object:Gem::Dependency
   name: pry-stack_explorer
-  requirement: &2161483620 !ruby/object:Gem::Requirement
+  requirement: &2161254620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2161483620
-description: Motivation for creating this was for queueing messages based on ID so
-  that very chatty message emitters don't prevent messages from others from being
-  processed
+  version_requirements: *2161254620
+description: A thread safe, Redis backed, parallel queue abstraction. Motivation for
+  creating this was for queueing messages based on ID so that very chatty message
+  emitters don't prevent messages from others from being processed
 email: daniel@populr.me
 executables: []
 extensions: []
@@ -128,7 +128,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1593146586268589980
+      hash: 1400047173269459282
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: